diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index b48889ef9d..9e6dd303c4 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -2745,7 +2745,240 @@ minillaplocal.query.files=\ temp_table_insert1_overwrite_partitions.q,\ temp_table_insert2_overwrite_partitions.q,\ temp_table_join1.q,\ - temp_table_load_dyn_part1.q + temp_table_load_dyn_part1.q,\ + temp_table_merge_dynamic_partition.q,\ + temp_table_merge_dynamic_partition2.q,\ + temp_table_merge_dynamic_partition3.q,\ + temp_table_merge_dynamic_partition4.q,\ + temp_table_merge_dynamic_partition5.q,\ + temp_table_options1.q,\ + temp_table_parquet_mixed_partition_formats2.q,\ + temp_table_partition_boolexpr.q,\ + temp_table_partition_condition_remover.q,\ + temp_table_partition_ctas.q,\ + temp_table_partition_multilevels.q,\ + temp_table_partition_pruning.q,\ + temp_table_windowing_expressions.q,\ + test_teradatabinaryfile.q,\ + timestamp.q,\ + timestamp_comparison3.q,\ + timestamp_ints_casts.q,\ + timestamp_literal.q,\ + timestamptz.q,\ + truncate_column_buckets.q,\ + truncate_column_list_bucket.q,\ + type_cast_1.q,\ + type_widening.q,\ + udaf_binarysetfunctions.q,\ + udaf_binarysetfunctions_no_cbo.q,\ + udaf_number_format.q,\ + udaf_percentile_cont.q,\ + udaf_percentile_disc.q,\ + udf1.q,\ + udf2.q,\ + udf3.q,\ + udf4.q,\ + udf5.q,\ + udf6.q,\ + udf7.q,\ + udf8.q,\ + udf9.q,\ + udf_10_trims.q,\ + udf_E.q,\ + udf_PI.q,\ + udf_abs.q,\ + udf_add_months.q,\ + udf_aes_decrypt.q,\ + udf_aes_encrypt.q,\ + udf_array.q,\ + udf_ascii.q,\ + udf_between.q,\ + udf_bitwise_shiftleft.q,\ + udf_bitwise_shiftright.q,\ + udf_bitwise_shiftrightunsigned.q,\ + udf_case.q,\ + udf_case_column_pruning.q,\ + udf_case_thrift.q,\ + udf_cbrt.q,\ + udf_character_length.q,\ + udf_concat_insert1.q,\ + udf_concat_ws.q,\ + udf_crc32.q,\ + udf_current_database.q,\ + udf_date_format.q,\ + udf_datetime_legacy_hybrid_calendar.q,\ + udf_decode.q,\ + udf_degrees.q,\ + udf_elt.q,\ + udf_example_add.q,\ + udf_explode.q,\ + udf_factorial.q,\ + udf_find_in_set.q,\ + udf_folder_constants.q,\ + udf_format_number.q,\ + udf_from_utc_timestamp.q,\ + udf_get_json_object.q,\ + udf_greatest.q,\ + udf_hash.q,\ + udf_hour.q,\ + udf_if.q,\ + udf_in_file.q,\ + udf_inline.q,\ + udf_instr.q,\ + udf_isnull_isnotnull.q,\ + udf_isops_simplify.q,\ + udf_java_method.q,\ + udf_last_day.q,\ + udf_least.q,\ + udf_length.q,\ + udf_levenshtein.q,\ + udf_like.q,\ + udf_locate.q,\ + udf_lower.q,\ + udf_lpad.q,\ + udf_map.q,\ + udf_mask.q,\ + udf_mask_first_n.q,\ + udf_mask_hash.q,\ + udf_mask_last_n.q,\ + udf_mask_show_first_n.q,\ + udf_mask_show_last_n.q,\ + udf_md5.q,\ + udf_minute.q,\ + udf_months_between.q,\ + udf_named_struct.q,\ + udf_next_day.q,\ + udf_notequal.q,\ + udf_nullif.q,\ + udf_nvl.q,\ + udf_octet_length.q,\ + udf_parse_url.q,\ + udf_position.q,\ + udf_quarter.q,\ + udf_radians.q,\ + udf_reflect.q,\ + udf_reflect2.q,\ + udf_repeat.q,\ + udf_reverse.q,\ + udf_rpad.q,\ + udf_second.q,\ + udf_sha1.q,\ + udf_sha2.q,\ + udf_sign.q,\ + udf_size.q,\ + udf_sort_array.q,\ + udf_sort_array_by.q,\ + udf_soundex.q,\ + udf_space.q,\ + udf_split.q,\ + udf_struct.q,\ + udf_substring_index.q,\ + udf_to_unix_timestamp.q,\ + udf_to_utc_timestamp.q,\ + udf_trunc.q,\ + udf_trunc_number.q,\ + udf_union.q,\ + udf_when.q,\ + udf_width_bucket.q,\ + udtf_explode.q,\ + udtf_get_sql_schema.q,\ + udtf_json_tuple.q,\ + udtf_parse_url_tuple.q,\ + udtf_stack.q,\ + unicode_comments.q,\ + unicode_data.q,\ + union.q,\ + union10.q,\ + union11.q,\ + union12.q,\ + union13.q,\ + union14.q,\ + union15.q,\ + union16.q,\ + union17.q,\ + union18.q,\ + union19.q,\ + union20.q,\ + union21.q,\ + union22.q,\ + union24.q,\ + union25.q,\ + union26.q,\ + union27.q,\ + union28.q,\ + union29.q,\ + union30.q,\ + union31.q,\ + union32.q,\ + union33.q,\ + union34.q,\ + union35.q,\ + union37.q,\ + union_lateralview.q,\ + union_offcbo.q,\ + union_paren.q,\ + union_pos_alias.q,\ + union_ppr.q,\ + union_remove_1.q,\ + union_remove_10.q,\ + union_remove_11.q,\ + union_remove_12.q,\ + union_remove_13.q,\ + union_remove_14.q,\ + union_remove_15.q,\ + union_remove_16.q,\ + union_remove_17.q,\ + union_remove_18.q,\ + union_remove_19.q,\ + union_remove_2.q,\ + union_remove_20.q,\ + union_remove_21.q,\ + union_remove_22.q,\ + union_remove_23.q,\ + union_remove_24.q,\ + union_remove_25.q,\ + union_remove_3.q,\ + union_remove_4.q,\ + union_remove_5.q,\ + union_remove_6.q,\ + union_remove_6_subq.q,\ + union_remove_7.q,\ + union_remove_8.q,\ + union_remove_9.q,\ + union_remove_plan.q,\ + union_view.q,\ + unionall_lateralview.q,\ + unionall_unbalancedppd.q,\ + updateBasicStats.q,\ + varchar_serde.q,\ + vector_const.q,\ + vector_decimal_col_scalar_division.q,\ + vector_decimal_partition.q,\ + vector_delete_orig_table.q,\ + vector_empty_where.q,\ + vector_gather_stats.q,\ + vector_non_constant_in_expr.q,\ + vector_outer_join_no_keys.q,\ + vector_tablesample_rows.q,\ + vector_windowing_row_number.q,\ + vectorization_multi_value.q,\ + vectorization_numeric_overflows.q,\ + vectorization_offset_limit.q,\ + vectorization_parquet_ppd_decimal.q,\ + vectorization_parquet_projection.q,\ + vectorization_sum_if_when.q,\ + vectorized_join46_mr.q,\ + vectorized_mapjoin2.q,\ + view_alias.q,\ + view_cbo.q,\ + windowing_expressions.q,\ + windowing_gby2.q,\ + windowing_multipartitioning.q,\ + windowing_navfn.q,\ + windowing_range_multiorder.q,\ + windowing_streaming.q,\ + windowing_udaf.q,\ + windowing_windowspec3.q encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ diff --git ql/src/test/queries/clientpositive/temp_table_options1.q ql/src/test/queries/clientpositive/temp_table_options1.q index b3362676c7..9a5bc64e70 100644 --- ql/src/test/queries/clientpositive/temp_table_options1.q +++ ql/src/test/queries/clientpositive/temp_table_options1.q @@ -1,3 +1,4 @@ +-- SORT_QUERY_RESULTS -- Delimiter test, taken from delimiter.q create temporary table impressions (imp string, msg string) row format delimited @@ -46,7 +47,7 @@ create temporary table date_serde_lb ( alter table date_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; insert overwrite table date_serde_lb - select fl_date, fl_num from date_serde_regex limit 1; + select fl_date, fl_num from date_serde_regex order by fl_date, fl_num limit 1; select * from date_serde_lb; select c1, sum(c2) from date_serde_lb group by c1; @@ -61,7 +62,7 @@ create temporary table date_serde_ls ( alter table date_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; insert overwrite table date_serde_ls - select c1, c2 from date_serde_lb limit 1; + select c1, c2 from date_serde_lb order by c1, c2 limit 1; select * from date_serde_ls; select c1, sum(c2) from date_serde_ls group by c1; @@ -76,7 +77,7 @@ create temporary table date_serde_c ( alter table date_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'; insert overwrite table date_serde_c - select c1, c2 from date_serde_ls limit 1; + select c1, c2 from date_serde_ls order by c1, c2 limit 1; select * from date_serde_c; select c1, sum(c2) from date_serde_c group by c1; @@ -91,7 +92,7 @@ create temporary table date_serde_lbc ( alter table date_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'; insert overwrite table date_serde_lbc - select c1, c2 from date_serde_c limit 1; + select c1, c2 from date_serde_c order by c1, c2 limit 1; select * from date_serde_lbc; select c1, sum(c2) from date_serde_lbc group by c1; @@ -106,7 +107,7 @@ create temporary table date_serde_orc ( alter table date_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'; insert overwrite table date_serde_orc - select c1, c2 from date_serde_lbc limit 1; + select c1, c2 from date_serde_lbc order by c1, c2 limit 1; select * from date_serde_orc; select c1, sum(c2) from date_serde_orc group by c1; diff --git ql/src/test/queries/clientpositive/temp_table_parquet_mixed_partition_formats2.q ql/src/test/queries/clientpositive/temp_table_parquet_mixed_partition_formats2.q index 9046460e37..af836314c8 100644 --- ql/src/test/queries/clientpositive/temp_table_parquet_mixed_partition_formats2.q +++ ql/src/test/queries/clientpositive/temp_table_parquet_mixed_partition_formats2.q @@ -17,17 +17,17 @@ OUTPUTFORMAT LOAD DATA LOCAL INPATH '../../data/files/sample2.json' INTO TABLE parquet_table_json_partition_temp PARTITION(ts='20150101'); -SELECT * FROM parquet_table_json_partition_temp LIMIT 100; +SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100; ALTER TABLE parquet_table_json_partition_temp SET FILEFORMAT INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'; -SELECT * FROM parquet_table_json_partition_temp LIMIT 100; +SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100; -CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp LIMIT 100; +CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100; -SELECT * FROM new_table_temp; +SELECT * FROM new_table_temp ORDER by id, address, reports; diff --git ql/src/test/queries/clientpositive/temp_table_partition_multilevels.q ql/src/test/queries/clientpositive/temp_table_partition_multilevels.q index 6232146969..50ba7c43ad 100644 --- ql/src/test/queries/clientpositive/temp_table_partition_multilevels.q +++ ql/src/test/queries/clientpositive/temp_table_partition_multilevels.q @@ -1,36 +1,38 @@ --! qt:dataset:srcpart +-- SORT_QUERY_RESULTS + set hive.mapred.mode=nonstrict; create temporary table partition_test_multilevel_temp (key string, value string) partitioned by (level1 string, level2 string, level3 string); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows); - -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows); - -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows); - -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows); - -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows); - -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows); -insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows); +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value; + +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value; + +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value; + +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value; + +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value; + +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value; +insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value; set metaconf:hive.metastore.try.direct.sql=false; diff --git ql/src/test/queries/clientpositive/udf_concat_insert1.q ql/src/test/queries/clientpositive/udf_concat_insert1.q index f735da6a66..eeef0580d9 100644 --- ql/src/test/queries/clientpositive/udf_concat_insert1.q +++ ql/src/test/queries/clientpositive/udf_concat_insert1.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + CREATE TABLE dest1_n139(key INT, value STRING) STORED AS TEXTFILE; FROM src diff --git ql/src/test/queries/clientpositive/udf_explode.q ql/src/test/queries/clientpositive/udf_explode.q index ec3f6daca4..7825eca50c 100644 --- ql/src/test/queries/clientpositive/udf_explode.q +++ ql/src/test/queries/clientpositive/udf_explode.q @@ -4,22 +4,22 @@ set hive.fetch.task.conversion=more; DESCRIBE FUNCTION explode; DESCRIBE FUNCTION EXTENDED explode; -EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows); -EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol; +EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol; +EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol; -SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows); -SELECT explode(array(1,2,3)) AS (myCol) FROM src tablesample (1 rows); -SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol; +SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol; +SELECT explode(array(1, 2, 3)) AS (myCol) FROM src tablesample (1 rows) ORDER BY myCol; +SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol; -EXPLAIN EXTENDED SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows); -EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val; +EXPLAIN EXTENDED SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val; +EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, value) a GROUP BY a.key, a.val ORDER BY a.key, a.val; -SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows); -SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val; +SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val; +SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val) a GROUP BY a.key, a.val ORDER BY a.key, a.val; drop table lazy_array_map; create table lazy_array_map (map_col map, array_col array); -INSERT OVERWRITE TABLE lazy_array_map select map(1,'one',2,'two',3,'three'), array('100','200','300') FROM src tablesample (1 rows); +INSERT OVERWRITE TABLE lazy_array_map select map(1, 'one', 2, 'two', 3, 'three'), array('100', '200', '300') FROM src tablesample (1 rows); -SELECT array_col, myCol from lazy_array_map lateral view explode(array_col) X AS myCol; -SELECT map_col, myKey, myValue from lazy_array_map lateral view explode(map_col) X AS myKey, myValue; \ No newline at end of file +SELECT array_col, myCol FROM lazy_array_map lateral view explode(array_col) X AS myCol ORDER BY array_col, myCol; +SELECT map_col, myKey, myValue FROM lazy_array_map lateral view explode(map_col) X AS myKey, myValue ORDER BY map_col, myKey, myValue; diff --git ql/src/test/queries/clientpositive/udf_sort_array_by.q ql/src/test/queries/clientpositive/udf_sort_array_by.q index 4c8c878424..d2ceb6e373 100644 --- ql/src/test/queries/clientpositive/udf_sort_array_by.q +++ ql/src/test/queries/clientpositive/udf_sort_array_by.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + use default; -- Test sort_array_by() UDF diff --git ql/src/test/queries/clientpositive/udf_width_bucket.q ql/src/test/queries/clientpositive/udf_width_bucket.q index 9fce6fc60c..324d9961e8 100644 --- ql/src/test/queries/clientpositive/udf_width_bucket.q +++ ql/src/test/queries/clientpositive/udf_width_bucket.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + describe function width_bucket; desc function extended width_bucket; diff --git ql/src/test/queries/clientpositive/udtf_explode.q ql/src/test/queries/clientpositive/udtf_explode.q index 5661d6cbbc..7bcc7cea1d 100644 --- ql/src/test/queries/clientpositive/udtf_explode.q +++ ql/src/test/queries/clientpositive/udtf_explode.q @@ -5,21 +5,21 @@ set hive.fetch.task.conversion=more; DESCRIBE FUNCTION explode; DESCRIBE FUNCTION EXTENDED explode; -EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3; -EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol; +EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3; +EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol; -SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3; -SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3; -SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol; +SELECT explode(array(1, 2, 3)) AS myCol FROM src ORDER BY myCol LIMIT 3; +SELECT explode(array(1, 2, 3)) AS (myCol) FROM src ORDER BY myCol LIMIT 3; +SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol ORDER BY a.myCol; -EXPLAIN SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3; -EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal; +EXPLAIN SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3; +EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal; -SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3; -SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal; +SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3; +SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal; -SELECT src.key, myCol FROM src lateral view explode(array(1,2,3)) x AS myCol LIMIT 3; -SELECT src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3; +SELECT src.key, myCol FROM src lateral view explode(array(1, 2, 3)) x AS myCol ORDER BY src.key, myCol LIMIT 3; +SELECT src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3; -- HIVE-4295 -SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3; +SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3; diff --git ql/src/test/queries/clientpositive/udtf_parse_url_tuple.q ql/src/test/queries/clientpositive/udtf_parse_url_tuple.q index 5e672b91dd..66aab52778 100644 --- ql/src/test/queries/clientpositive/udtf_parse_url_tuple.q +++ ql/src/test/queries/clientpositive/udtf_parse_url_tuple.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + set hive.mapred.mode=nonstrict; create table url_t (key string, fullurl string); diff --git ql/src/test/queries/clientpositive/union.q ql/src/test/queries/clientpositive/union.q index 3f40a25d49..5d50be3837 100644 --- ql/src/test/queries/clientpositive/union.q +++ ql/src/test/queries/clientpositive/union.q @@ -1,6 +1,5 @@ --! qt:dataset:src set hive.mapred.mode=nonstrict; --- SORT_BEFORE_DIFF -- union case: both subqueries are map jobs on same input, followed by filesink EXPLAIN @@ -9,13 +8,13 @@ FROM ( UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.*; +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value; FROM ( FROM src select src.key, src.value WHERE src.key < 100 UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.*; +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value; dfs -cat ${system:test.warehouse.dir}/union.out/*; diff --git ql/src/test/queries/clientpositive/union13.q ql/src/test/queries/clientpositive/union13.q index 6b8608ea6b..826c96e6f2 100644 --- ql/src/test/queries/clientpositive/union13.q +++ ql/src/test/queries/clientpositive/union13.q @@ -1,6 +1,6 @@ --! qt:dataset:src set hive.mapred.mode=nonstrict; --- SORT_BEFORE_DIFF +-- SORT_QUERY_RESULTS -- union case: both subqueries are a map-only jobs, same input, followed by filesink explain diff --git ql/src/test/queries/clientpositive/union15.q ql/src/test/queries/clientpositive/union15.q index d2590f8307..70524f29f9 100644 --- ql/src/test/queries/clientpositive/union15.q +++ ql/src/test/queries/clientpositive/union15.q @@ -1,8 +1,9 @@ --! qt:dataset:src1 --! qt:dataset:src +-- SORT_QUERY_RESULTS + set hive.mapred.mode=nonstrict; set hive.map.aggr = true; --- SORT_BEFORE_DIFF -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink explain diff --git ql/src/test/queries/clientpositive/union35.q ql/src/test/queries/clientpositive/union35.q index 38e9561de6..99d271c210 100644 --- ql/src/test/queries/clientpositive/union35.q +++ ql/src/test/queries/clientpositive/union35.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + select * from ( select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f union all diff --git ql/src/test/queries/clientpositive/union_pos_alias.q ql/src/test/queries/clientpositive/union_pos_alias.q index fd5c19f060..d881466a7d 100644 --- ql/src/test/queries/clientpositive/union_pos_alias.q +++ ql/src/test/queries/clientpositive/union_pos_alias.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + set hive.mapred.mode=nonstrict; @@ -14,7 +16,7 @@ select key, value from (select 'tst2' as key, count(1) as value from src s2 UNIO order by 1; drop table src_10; -create table src_10 as select * from src limit 10; +create table src_10 as select * from src order by key, value limit 10; explain select key as value, value as key from src_10 diff --git ql/src/test/queries/clientpositive/varchar_serde.q ql/src/test/queries/clientpositive/varchar_serde.q index 7b3d7a90a2..426b09c76a 100644 --- ql/src/test/queries/clientpositive/varchar_serde.q +++ ql/src/test/queries/clientpositive/varchar_serde.q @@ -22,8 +22,8 @@ stored as textfile; load data local inpath '../../data/files/srcbucket0.txt' overwrite into table varchar_serde_regex; -select * from varchar_serde_regex limit 5; -select value, count(*) from varchar_serde_regex group by value limit 5; +select * from varchar_serde_regex order by key, value limit 5; +select value, count(*) from varchar_serde_regex group by value order by value limit 5; -- -- LazyBinary @@ -36,8 +36,8 @@ alter table varchar_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary insert overwrite table varchar_serde_lb select key, value from varchar_serde_regex; -select * from varchar_serde_lb limit 5; -select value, count(*) from varchar_serde_lb group by value limit 5; +select * from varchar_serde_lb order by key, value limit 5; +select value, count(*) from varchar_serde_lb group by value order by value limit 5; -- -- LazySimple @@ -50,8 +50,8 @@ alter table varchar_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazyS insert overwrite table varchar_serde_ls select key, value from varchar_serde_lb; -select * from varchar_serde_ls limit 5; -select value, count(*) from varchar_serde_ls group by value limit 5; +select * from varchar_serde_ls order by key, value limit 5; +select value, count(*) from varchar_serde_ls group by value order by value limit 5; -- -- Columnar @@ -64,8 +64,8 @@ alter table varchar_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.Co insert overwrite table varchar_serde_c select key, value from varchar_serde_ls; -select * from varchar_serde_c limit 5; -select value, count(*) from varchar_serde_c group by value limit 5; +select * from varchar_serde_c order by key, value limit 5; +select value, count(*) from varchar_serde_c group by value order by value limit 5; -- -- LazyBinaryColumnar @@ -78,8 +78,8 @@ alter table varchar_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar. insert overwrite table varchar_serde_lbc select key, value from varchar_serde_c; -select * from varchar_serde_lbc limit 5; -select value, count(*) from varchar_serde_lbc group by value limit 5; +select * from varchar_serde_lbc order by key, value limit 5; +select value, count(*) from varchar_serde_lbc group by value order by value limit 5; -- -- ORC @@ -93,8 +93,8 @@ alter table varchar_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSer insert overwrite table varchar_serde_orc select key, value from varchar_serde_lbc; -select * from varchar_serde_orc limit 5; -select value, count(*) from varchar_serde_orc group by value limit 5; +select * from varchar_serde_orc order by key, value limit 5; +select value, count(*) from varchar_serde_orc group by value order by value limit 5; drop table if exists varchar_serde_regex; drop table if exists varchar_serde_lb; diff --git ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q index 02f4739a1a..4e1fcf0347 100644 --- ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q +++ ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q @@ -10,7 +10,12 @@ set hive.llap.cache.allow.synthetic.fileid=true; create table newtypestbl_n1(c char(10), v varchar(10), d decimal(5,3), da date) stored as parquet; -insert overwrite table newtypestbl_n1 select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 10) uniontbl; +insert overwrite table newtypestbl_n1 +select * + from (select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 limit 5) u1 + union all + select * from (select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 5) u2 + ) uniontbl; -- decimal data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select * from newtypestbl_n1 where d=0.22; diff --git ql/src/test/queries/clientpositive/view_alias.q ql/src/test/queries/clientpositive/view_alias.q index 1f3f3b165b..6f5fd364f4 100644 --- ql/src/test/queries/clientpositive/view_alias.q +++ ql/src/test/queries/clientpositive/view_alias.q @@ -1,8 +1,8 @@ --! qt:dataset:src drop view v_n6; -create view v_n6 as select key, '12' from src; +create view v_n6 as select 10 - key, '12' from src; desc formatted v_n6; -select * from v_n6 order by `_c1` limit 5; +select * from v_n6 order by `_c0`, `_c1` limit 5; drop view v_n6; create view v_n6 as select key as `_c1`, '12' from src; @@ -12,24 +12,24 @@ select * from v_n6 order by `_c1` limit 5; drop view v_n6; create view v_n6 as select *, '12' from src; desc formatted v_n6; -select * from v_n6 order by `_c2` limit 5; +select * from v_n6 order by key, value, `_c2` limit 5; drop view v_n6; create view v_n6 as select *, '12' as `_c121` from src; desc formatted v_n6; -select * from v_n6 order by `_c121` limit 5; +select * from v_n6 order by key, value, `_c121` limit 5; drop view v_n6; create view v_n6 as select key, count(*) from src group by key; desc formatted v_n6; -select * from v_n6 order by `_c1` limit 5; +select * from v_n6 order by key, `_c1` limit 5; drop view v_n6; create table a_n9 (ca_n9 string, caa_n9 string); create table b_n7 (cb_n7 string, cbb_n7 string); -insert into a_n9 select * from src limit 5; -insert into b_n7 select * from src limit 5; +insert into a_n9 select * from src order by key, value limit 5; +insert into b_n7 select * from src order by key, value limit 5; create view v_n6 as select '010', a_n9.*, 121, b_n7.*, 234 from a_n9 join b_n7 on a_n9.ca_n9 = b_n7.cb_n7; desc formatted v_n6; -select * from v_n6 order by `_c3` limit 5; +select * from v_n6 order by `_c3`, `_c0`, ca_n9, caa_n9, cb_n7, cbb_n7 limit 5; diff --git ql/src/test/queries/clientpositive/windowing_expressions.q ql/src/test/queries/clientpositive/windowing_expressions.q index 09e759b598..855257b240 100644 --- ql/src/test/queries/clientpositive/windowing_expressions.q +++ ql/src/test/queries/clientpositive/windowing_expressions.q @@ -31,10 +31,10 @@ sum(p_retailprice) over (distribute by p_mfgr sort by p_retailprice rows between from part ; -select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 limit 100; -select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 limit 100; -select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 limit 100; -select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 limit 100; +select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 order by s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) limit 100; +select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 order by s, i, i - lead(i, 3, 0) over (partition by si order by i,s)limit 100; +select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 order by s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) limit 100; +select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 order by s, lag(s, 3, 'fred') over (partition by f order by b) limit 100; select p_mfgr, avg(p_retailprice) over(partition by p_mfgr, p_type order by p_mfgr) from part; @@ -44,11 +44,12 @@ select p_mfgr, avg(p_retailprice) over(partition by p_mfgr order by p_type,p_mfg create table t1_n142 (a1 int, b1 string); create table t2_n83 (a1 int, b1 string); from (select sum(i) over (partition by ts order by i), s from over10k_n22) tt insert overwrite table t1_n142 select * insert overwrite table t2_n83 select * ; -select * from t1_n142 limit 3; -select * from t2_n83 limit 3; +select * from t1_n142 order by a1, b1 limit 3; +select * from t2_n83 order by a1, b1 limit 3; select p_mfgr, p_retailprice, p_size, round(sum(p_retailprice) over w1 , 2) + 50.0 = round(sum(lag(p_retailprice,1,50.0)) over w1 + (last_value(p_retailprice) over w1),2) from part window w1 as (distribute by p_mfgr sort by p_retailprice) +order by p_mfgr, p_retailprice, p_size limit 11; diff --git ql/src/test/queries/clientpositive/windowing_multipartitioning.q ql/src/test/queries/clientpositive/windowing_multipartitioning.q index 622c244365..2a23d56f0b 100644 --- ql/src/test/queries/clientpositive/windowing_multipartitioning.q +++ ql/src/test/queries/clientpositive/windowing_multipartitioning.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + drop table over10k_n11; create table over10k_n11( @@ -17,7 +19,7 @@ create table over10k_n11( load data local inpath '../../data/files/over10k' into table over10k_n11; -select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 limit 100; +select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 order by s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) limit 100; select s, rank() over (partition by s order by `dec` desc), diff --git ql/src/test/queries/clientpositive/windowing_range_multiorder.q ql/src/test/queries/clientpositive/windowing_range_multiorder.q index a09c717881..8858466134 100644 --- ql/src/test/queries/clientpositive/windowing_range_multiorder.q +++ ql/src/test/queries/clientpositive/windowing_range_multiorder.q @@ -17,24 +17,24 @@ create table over10k_n17( load data local inpath '../../data/files/over10k' into table over10k_n17; -select first_value(t) over ( partition by si order by i, b ) from over10k_n17 limit 100; +select first_value(t) over ( partition by si order by i, b ) fv from over10k_n17 order by fv limit 100; -select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k_n17 limit 100; +select last_value(i) over (partition by si, bo order by i, f desc range current row) lv from over10k_n17 order by lv limit 100; -select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k_n17 limit 100; +select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) rn from over10k_n17 order by rn limit 100; -select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k_n17 limit 100; +select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100; -select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100; +select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100; -select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100; +select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) m from over10k_n17 order by s, si, i, m limit 100; -select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k_n17 limit 100; +select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100; -select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k_n17 limit 100; +select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) m from over10k_n17 order by si, bo, i, f, m limit 100; -select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100; +select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by bo, r limit 100; -select CAST(s as CHAR(12)), rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100; +select CAST(s as CHAR(12)) s, rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100; -select CAST(s as VARCHAR(12)), rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100; +select CAST(s as VARCHAR(12)) s, rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100; diff --git ql/src/test/queries/clientpositive/windowing_udaf.q ql/src/test/queries/clientpositive/windowing_udaf.q index 2e7da0d2e9..3aab200d93 100644 --- ql/src/test/queries/clientpositive/windowing_udaf.q +++ ql/src/test/queries/clientpositive/windowing_udaf.q @@ -18,16 +18,17 @@ create table over10k_n4( load data local inpath '../../data/files/over10k' into table over10k_n4; -select s, min(i) over (partition by s) from over10k_n4 limit 100; +select s, min(i) over (partition by s) m from over10k_n4 order by s, m limit 100; -select s, avg(f) over (partition by si order by s) from over10k_n4 limit 100; +select s, avg(f) over (partition by si order by s) a from over10k_n4 order by s, a limit 100; -select s, avg(i) over (partition by t, b order by s) from over10k_n4 limit 100; +select s, avg(i) over (partition by t, b order by s) a from over10k_n4 order by s, a limit 100; -select max(i) over w from over10k_n4 window w as (partition by f) limit 100; +select max(i) over w m from over10k_n4 window w as (partition by f) order by m limit 100; -select s, avg(d) over (partition by t order by f) from over10k_n4 limit 100; +select s, avg(d) over (partition by t order by f) a from over10k_n4 order by s, a limit 100; select key, max(value) over - (order by key rows between 10 preceding and 20 following) -from src1 where length(key) > 10; + (order by key rows between 10 preceding and 20 following) m +from src1 where length(key) > 10 +order by key, m; diff --git ql/src/test/queries/clientpositive/windowing_windowspec3.q ql/src/test/queries/clientpositive/windowing_windowspec3.q index debd076b72..fac52b68ac 100644 --- ql/src/test/queries/clientpositive/windowing_windowspec3.q +++ ql/src/test/queries/clientpositive/windowing_windowspec3.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + -- Test value based windowing spec drop table if exists emp_n0; diff --git ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition.q.out similarity index 84% rename from ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition.q.out index 8b1cfad5ab..abb4f779eb 100644 --- ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition.q.out @@ -60,41 +60,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_n1_temp POSTHOOK: Input: default@srcpart_merge_dp_n1_temp@ds=2008-04-08/hr=11 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_n1_temp - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_n1_temp + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n1_temp + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -109,7 +123,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part_n1_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: @@ -666,79 +680,79 @@ POSTHOOK: Input: default@srcpart_merge_dp_n1_temp@ds=2008-04-08/hr=11 POSTHOOK: Output: default@merge_dynamic_part_n1_temp@ds=2008-04-08/hr=11 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_n1_temp - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_n1_temp + filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator + File Output Operator + compressed: false + Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n1_temp + Select Operator + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string) + outputColumnNames: key, value, ds, hr + Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string), hr (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 99 Data size: 31648 Basic stats: PARTIAL Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 49 Data size: 15664 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -753,7 +767,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part_n1_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -761,36 +775,6 @@ STAGE PLANS: Column Types: string, string Table: default.merge_dynamic_part_n1_temp - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_n1_temp partition (ds='2008-04-08', hr=11) select key, value from srcpart_merge_dp_n1_temp where ds='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart_merge_dp_n1_temp @@ -1343,55 +1327,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_n1_temp POSTHOOK: Input: default@srcpart_merge_dp_n1_temp@ds=2008-04-08/hr=11 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_n1_temp - filterExpr: ((ds = '2008-04-08') and (11.0D = 11.0D)) (type: boolean) - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_n1_temp + filterExpr: ((ds = '2008-04-08') and (11.0D = 11.0D)) (type: boolean) + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 99 Data size: 49864 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n1_temp + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -1406,40 +1390,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part_n1_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1_temp - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_n1_temp partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n1_temp where ds='2008-04-08' and hr=11 PREHOOK: type: QUERY PREHOOK: Input: default@srcpart_merge_dp_n1_temp diff --git ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition2.q.out ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition2.q.out similarity index 72% rename from ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition2.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition2.q.out index 413a3f2a63..c1fb807b9d 100644 --- ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition2.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition2.q.out @@ -79,55 +79,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_n0_temp@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart_merge_dp_n0_temp@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_n0_temp - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n0_temp + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_n0_temp + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 297 Data size: 148488 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n0_temp - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -142,40 +142,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part_n0_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n0_temp - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n0_temp - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_n0_temp partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp_n0_temp where ds='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart_merge_dp_n0_temp diff --git ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition3.q.out ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition3.q.out similarity index 81% rename from ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition3.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition3.q.out index 12d5d59fb1..9d6669d266 100644 --- ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition3.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition3.q.out @@ -143,55 +143,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_n2_temp@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart_merge_dp_n2_temp@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_n2_temp - filterExpr: (ds >= '2008-04-08') (type: boolean) - Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL - Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL - Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n2_temp - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_n2_temp + filterExpr: (ds >= '2008-04-08') (type: boolean) + Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 594 Data size: 405536 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n2_temp + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -206,40 +206,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_dynamic_part_n2_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n2_temp - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n2_temp - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_n2_temp partition (ds, hr) select key, value, ds, hr from srcpart_merge_dp_n2_temp where ds>='2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart_merge_dp_n2_temp diff --git ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition4.q.out ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition4.q.out similarity index 83% rename from ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition4.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition4.q.out index 8ddbb96fca..940b065142 100644 --- ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition4.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition4.q.out @@ -140,55 +140,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_rc_n1_temp@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart_merge_dp_rc_n1_temp@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_rc_n1_temp - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 2.0D) = 0.0D), 'a1', 'b1') (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.merge_dynamic_part_n3_temp + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_rc_n1_temp + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 2.0D) = 0.0D), 'a1', 'b1') (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 349968 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part_n3_temp - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -203,30 +203,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.merge_dynamic_part_n3_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_n3_temp partition (ds = '2008-04-08', hr) select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc_n1_temp where ds = '2008-04-08' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition5.q.out ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition5.q.out similarity index 80% rename from ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition5.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition5.q.out index 7dbf56cf39..3718eef72e 100644 --- ql/src/test/results/clientpositive/temp_table_merge_dynamic_partition5.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_merge_dynamic_partition5.q.out @@ -116,55 +116,55 @@ POSTHOOK: Input: default@srcpart_merge_dp_rc_temp@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart_merge_dp_rc_temp@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart_merge_dp_rc_temp - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.merge_dynamic_part_temp - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_merge_dp_rc_temp + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 618 Data size: 216752 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part_temp + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -179,30 +179,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.merge_dynamic_part_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table merge_dynamic_part_temp partition (ds = '2008-04-08', hr) select key, value, if(key % 100 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc_temp where ds = '2008-04-08' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/temp_table_options1.q.out ql/src/test/results/clientpositive/llap/temp_table_options1.q.out similarity index 96% rename from ql/src/test/results/clientpositive/temp_table_options1.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_options1.q.out index be31a5a289..3ca126ba7a 100644 --- ql/src/test/results/clientpositive/temp_table_options1.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_options1.q.out @@ -31,8 +31,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@impressions #### A masked pattern was here #### 35 40 -48 32 100100 40 +48 32 PREHOOK: query: select imp,msg from impressions PREHOOK: type: QUERY PREHOOK: Input: default@impressions @@ -42,8 +42,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@impressions #### A masked pattern was here #### 35 40 -48 32 100100 40 +48 32 PREHOOK: query: drop table impressions PREHOOK: type: DROPTABLE PREHOOK: Input: default@impressions @@ -101,139 +101,139 @@ POSTHOOK: Input: default@date_serde_regex Baltimore New York 2010-10-20 -30.0 1064 Baltimore New York 2010-10-20 23.0 1142 Baltimore New York 2010-10-20 6.0 1599 -Chicago New York 2010-10-20 42.0 361 -Chicago New York 2010-10-20 24.0 897 -Chicago New York 2010-10-20 15.0 1531 -Chicago New York 2010-10-20 -6.0 1610 -Chicago New York 2010-10-20 -2.0 3198 -Baltimore New York 2010-10-21 17.0 1064 Baltimore New York 2010-10-21 105.0 1142 +Baltimore New York 2010-10-21 17.0 1064 Baltimore New York 2010-10-21 28.0 1599 -Chicago New York 2010-10-21 142.0 361 -Chicago New York 2010-10-21 77.0 897 -Chicago New York 2010-10-21 53.0 1531 -Chicago New York 2010-10-21 -5.0 1610 -Chicago New York 2010-10-21 51.0 3198 Baltimore New York 2010-10-22 -12.0 1064 -Baltimore New York 2010-10-22 54.0 1142 Baltimore New York 2010-10-22 18.0 1599 -Chicago New York 2010-10-22 2.0 361 -Chicago New York 2010-10-22 24.0 897 -Chicago New York 2010-10-22 16.0 1531 -Chicago New York 2010-10-22 -6.0 1610 -Chicago New York 2010-10-22 -11.0 3198 -Baltimore New York 2010-10-23 18.0 272 +Baltimore New York 2010-10-22 54.0 1142 Baltimore New York 2010-10-23 -10.0 1805 +Baltimore New York 2010-10-23 18.0 272 Baltimore New York 2010-10-23 6.0 3171 -Chicago New York 2010-10-23 3.0 384 -Chicago New York 2010-10-23 32.0 426 -Chicago New York 2010-10-23 1.0 650 -Chicago New York 2010-10-23 11.0 3085 Baltimore New York 2010-10-24 12.0 1599 Baltimore New York 2010-10-24 20.0 2571 -Chicago New York 2010-10-24 10.0 361 -Chicago New York 2010-10-24 113.0 897 -Chicago New York 2010-10-24 -5.0 1531 -Chicago New York 2010-10-24 -17.0 1610 -Chicago New York 2010-10-24 -3.0 3198 Baltimore New York 2010-10-25 -25.0 1064 -Baltimore New York 2010-10-25 92.0 1142 Baltimore New York 2010-10-25 106.0 1599 -Chicago New York 2010-10-25 31.0 361 -Chicago New York 2010-10-25 -1.0 897 -Chicago New York 2010-10-25 43.0 1531 -Chicago New York 2010-10-25 6.0 1610 -Chicago New York 2010-10-25 -16.0 3198 +Baltimore New York 2010-10-25 92.0 1142 Baltimore New York 2010-10-26 -22.0 1064 Baltimore New York 2010-10-26 123.0 1142 Baltimore New York 2010-10-26 90.0 1599 -Chicago New York 2010-10-26 12.0 361 -Chicago New York 2010-10-26 0.0 897 -Chicago New York 2010-10-26 29.0 1531 -Chicago New York 2010-10-26 -17.0 1610 -Chicago New York 2010-10-26 6.0 3198 Baltimore New York 2010-10-27 -18.0 1064 Baltimore New York 2010-10-27 49.0 1142 Baltimore New York 2010-10-27 92.0 1599 -Chicago New York 2010-10-27 148.0 361 +Baltimore New York 2010-10-28 -14.0 1142 +Baltimore New York 2010-10-28 -14.0 1599 +Baltimore New York 2010-10-28 -4.0 1064 +Baltimore New York 2010-10-29 -2.0 1599 +Baltimore New York 2010-10-29 -24.0 1064 +Baltimore New York 2010-10-29 21.0 1142 +Baltimore New York 2010-10-30 -1.0 1805 +Baltimore New York 2010-10-30 14.0 272 +Baltimore New York 2010-10-30 5.0 3171 +Baltimore New York 2010-10-31 -1.0 1599 +Baltimore New York 2010-10-31 -14.0 2571 +Chicago New York 2010-10-20 -2.0 3198 +Chicago New York 2010-10-20 -6.0 1610 +Chicago New York 2010-10-20 15.0 1531 +Chicago New York 2010-10-20 24.0 897 +Chicago New York 2010-10-20 42.0 361 +Chicago New York 2010-10-21 -5.0 1610 +Chicago New York 2010-10-21 142.0 361 +Chicago New York 2010-10-21 51.0 3198 +Chicago New York 2010-10-21 53.0 1531 +Chicago New York 2010-10-21 77.0 897 +Chicago New York 2010-10-22 -11.0 3198 +Chicago New York 2010-10-22 -6.0 1610 +Chicago New York 2010-10-22 16.0 1531 +Chicago New York 2010-10-22 2.0 361 +Chicago New York 2010-10-22 24.0 897 +Chicago New York 2010-10-23 1.0 650 +Chicago New York 2010-10-23 11.0 3085 +Chicago New York 2010-10-23 3.0 384 +Chicago New York 2010-10-23 32.0 426 +Chicago New York 2010-10-24 -17.0 1610 +Chicago New York 2010-10-24 -3.0 3198 +Chicago New York 2010-10-24 -5.0 1531 +Chicago New York 2010-10-24 10.0 361 +Chicago New York 2010-10-24 113.0 897 +Chicago New York 2010-10-25 -1.0 897 +Chicago New York 2010-10-25 -16.0 3198 +Chicago New York 2010-10-25 31.0 361 +Chicago New York 2010-10-25 43.0 1531 +Chicago New York 2010-10-25 6.0 1610 +Chicago New York 2010-10-26 -17.0 1610 +Chicago New York 2010-10-26 0.0 897 +Chicago New York 2010-10-26 12.0 361 +Chicago New York 2010-10-26 29.0 1531 +Chicago New York 2010-10-26 6.0 3198 Chicago New York 2010-10-27 -11.0 897 +Chicago New York 2010-10-27 148.0 361 +Chicago New York 2010-10-27 21.0 3198 Chicago New York 2010-10-27 70.0 1531 Chicago New York 2010-10-27 8.0 1610 -Chicago New York 2010-10-27 21.0 3198 -Baltimore New York 2010-10-28 -4.0 1064 -Baltimore New York 2010-10-28 -14.0 1142 -Baltimore New York 2010-10-28 -14.0 1599 +Chicago New York 2010-10-28 -11.0 1531 +Chicago New York 2010-10-28 -18.0 3198 Chicago New York 2010-10-28 2.0 361 Chicago New York 2010-10-28 2.0 897 -Chicago New York 2010-10-28 -11.0 1531 Chicago New York 2010-10-28 3.0 1610 -Chicago New York 2010-10-28 -18.0 3198 -Baltimore New York 2010-10-29 -24.0 1064 -Baltimore New York 2010-10-29 21.0 1142 -Baltimore New York 2010-10-29 -2.0 1599 -Chicago New York 2010-10-29 -12.0 361 Chicago New York 2010-10-29 -11.0 897 -Chicago New York 2010-10-29 15.0 1531 +Chicago New York 2010-10-29 -12.0 361 Chicago New York 2010-10-29 -18.0 1610 Chicago New York 2010-10-29 -4.0 3198 -Baltimore New York 2010-10-30 14.0 272 -Baltimore New York 2010-10-30 -1.0 1805 -Baltimore New York 2010-10-30 5.0 3171 -Chicago New York 2010-10-30 -6.0 384 +Chicago New York 2010-10-29 15.0 1531 Chicago New York 2010-10-30 -10.0 426 -Chicago New York 2010-10-30 -5.0 650 Chicago New York 2010-10-30 -5.0 3085 -Baltimore New York 2010-10-31 -1.0 1599 -Baltimore New York 2010-10-31 -14.0 2571 -Chicago New York 2010-10-31 -25.0 361 +Chicago New York 2010-10-30 -5.0 650 +Chicago New York 2010-10-30 -6.0 384 +Chicago New York 2010-10-31 -15.0 3198 Chicago New York 2010-10-31 -18.0 897 -Chicago New York 2010-10-31 -4.0 1531 Chicago New York 2010-10-31 -22.0 1610 -Chicago New York 2010-10-31 -15.0 3198 -Cleveland New York 2010-10-30 -23.0 2018 -Cleveland New York 2010-10-30 -12.0 2932 -Cleveland New York 2010-10-29 -4.0 2630 -Cleveland New York 2010-10-29 -19.0 2646 -Cleveland New York 2010-10-29 -12.0 3014 -Cleveland New York 2010-10-28 3.0 2630 -Cleveland New York 2010-10-28 -6.0 2646 -Cleveland New York 2010-10-28 1.0 3014 -Cleveland New York 2010-10-27 16.0 2630 -Cleveland New York 2010-10-27 27.0 3014 -Cleveland New York 2010-10-26 4.0 2630 -Cleveland New York 2010-10-26 -27.0 2646 -Cleveland New York 2010-10-26 -11.0 2662 -Cleveland New York 2010-10-26 13.0 3014 -Cleveland New York 2010-10-25 -4.0 2630 -Cleveland New York 2010-10-25 81.0 2646 -Cleveland New York 2010-10-25 42.0 3014 -Cleveland New York 2010-10-24 5.0 2254 +Chicago New York 2010-10-31 -25.0 361 +Chicago New York 2010-10-31 -4.0 1531 +Cleveland New York 2010-10-20 -15.0 3014 +Cleveland New York 2010-10-20 -8.0 2630 +Cleveland New York 2010-10-21 29.0 2646 +Cleveland New York 2010-10-21 3.0 2630 +Cleveland New York 2010-10-21 72.0 3014 +Cleveland New York 2010-10-22 -25.0 2646 +Cleveland New York 2010-10-22 -3.0 3014 +Cleveland New York 2010-10-22 1.0 2630 +Cleveland New York 2010-10-23 -21.0 2932 Cleveland New York 2010-10-24 -11.0 2630 Cleveland New York 2010-10-24 -20.0 2646 Cleveland New York 2010-10-24 -9.0 3014 -Cleveland New York 2010-10-23 -21.0 2932 -Cleveland New York 2010-10-22 1.0 2630 -Cleveland New York 2010-10-22 -25.0 2646 -Cleveland New York 2010-10-22 -3.0 3014 -Cleveland New York 2010-10-21 3.0 2630 -Cleveland New York 2010-10-21 29.0 2646 -Cleveland New York 2010-10-21 72.0 3014 -Cleveland New York 2010-10-20 -8.0 2630 -Cleveland New York 2010-10-20 -15.0 3014 -Washington New York 2010-10-23 -25.0 5832 -Washington New York 2010-10-23 -21.0 5904 -Washington New York 2010-10-23 -18.0 5917 -Washington New York 2010-10-30 -27.0 5904 -Washington New York 2010-10-30 -16.0 5917 +Cleveland New York 2010-10-24 5.0 2254 +Cleveland New York 2010-10-25 -4.0 2630 +Cleveland New York 2010-10-25 42.0 3014 +Cleveland New York 2010-10-25 81.0 2646 +Cleveland New York 2010-10-26 -11.0 2662 +Cleveland New York 2010-10-26 -27.0 2646 +Cleveland New York 2010-10-26 13.0 3014 +Cleveland New York 2010-10-26 4.0 2630 +Cleveland New York 2010-10-27 16.0 2630 +Cleveland New York 2010-10-27 27.0 3014 +Cleveland New York 2010-10-28 -6.0 2646 +Cleveland New York 2010-10-28 1.0 3014 +Cleveland New York 2010-10-28 3.0 2630 +Cleveland New York 2010-10-29 -12.0 3014 +Cleveland New York 2010-10-29 -19.0 2646 +Cleveland New York 2010-10-29 -4.0 2630 +Cleveland New York 2010-10-30 -12.0 2932 +Cleveland New York 2010-10-30 -23.0 2018 Washington New York 2010-10-20 -2.0 7291 Washington New York 2010-10-21 22.0 7291 Washington New York 2010-10-23 -16.0 7274 +Washington New York 2010-10-23 -18.0 5917 +Washington New York 2010-10-23 -21.0 5904 +Washington New York 2010-10-23 -25.0 5832 Washington New York 2010-10-24 -26.0 7282 Washington New York 2010-10-25 9.0 7291 Washington New York 2010-10-26 4.0 7291 Washington New York 2010-10-27 26.0 7291 Washington New York 2010-10-28 45.0 7291 Washington New York 2010-10-29 1.0 7291 +Washington New York 2010-10-30 -16.0 5917 +Washington New York 2010-10-30 -27.0 5904 Washington New York 2010-10-31 -18.0 7282 PREHOOK: query: select fl_date, count(*) from date_serde_regex group by fl_date PREHOOK: type: QUERY @@ -278,12 +278,12 @@ POSTHOOK: type: ALTERTABLE_SERIALIZER POSTHOOK: Input: default@date_serde_lb POSTHOOK: Output: default@date_serde_lb PREHOOK: query: insert overwrite table date_serde_lb - select fl_date, fl_num from date_serde_regex limit 1 + select fl_date, fl_num from date_serde_regex order by fl_date, fl_num limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_regex PREHOOK: Output: default@date_serde_lb POSTHOOK: query: insert overwrite table date_serde_lb - select fl_date, fl_num from date_serde_regex limit 1 + select fl_date, fl_num from date_serde_regex order by fl_date, fl_num limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_regex POSTHOOK: Output: default@date_serde_lb @@ -297,7 +297,7 @@ POSTHOOK: query: select * from date_serde_lb POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lb #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: select c1, sum(c2) from date_serde_lb group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_lb @@ -306,7 +306,7 @@ POSTHOOK: query: select c1, sum(c2) from date_serde_lb group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lb #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: create temporary table date_serde_ls ( c1 date, c2 int @@ -330,12 +330,12 @@ POSTHOOK: type: ALTERTABLE_SERIALIZER POSTHOOK: Input: default@date_serde_ls POSTHOOK: Output: default@date_serde_ls PREHOOK: query: insert overwrite table date_serde_ls - select c1, c2 from date_serde_lb limit 1 + select c1, c2 from date_serde_lb order by c1, c2 limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_lb PREHOOK: Output: default@date_serde_ls POSTHOOK: query: insert overwrite table date_serde_ls - select c1, c2 from date_serde_lb limit 1 + select c1, c2 from date_serde_lb order by c1, c2 limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lb POSTHOOK: Output: default@date_serde_ls @@ -349,7 +349,7 @@ POSTHOOK: query: select * from date_serde_ls POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_ls #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: select c1, sum(c2) from date_serde_ls group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_ls @@ -358,7 +358,7 @@ POSTHOOK: query: select c1, sum(c2) from date_serde_ls group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_ls #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: create temporary table date_serde_c ( c1 date, c2 int @@ -382,12 +382,12 @@ POSTHOOK: type: ALTERTABLE_SERIALIZER POSTHOOK: Input: default@date_serde_c POSTHOOK: Output: default@date_serde_c PREHOOK: query: insert overwrite table date_serde_c - select c1, c2 from date_serde_ls limit 1 + select c1, c2 from date_serde_ls order by c1, c2 limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_ls PREHOOK: Output: default@date_serde_c POSTHOOK: query: insert overwrite table date_serde_c - select c1, c2 from date_serde_ls limit 1 + select c1, c2 from date_serde_ls order by c1, c2 limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_ls POSTHOOK: Output: default@date_serde_c @@ -401,7 +401,7 @@ POSTHOOK: query: select * from date_serde_c POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_c #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: select c1, sum(c2) from date_serde_c group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_c @@ -410,7 +410,7 @@ POSTHOOK: query: select c1, sum(c2) from date_serde_c group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_c #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: create temporary table date_serde_lbc ( c1 date, c2 int @@ -434,12 +434,12 @@ POSTHOOK: type: ALTERTABLE_SERIALIZER POSTHOOK: Input: default@date_serde_lbc POSTHOOK: Output: default@date_serde_lbc PREHOOK: query: insert overwrite table date_serde_lbc - select c1, c2 from date_serde_c limit 1 + select c1, c2 from date_serde_c order by c1, c2 limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_c PREHOOK: Output: default@date_serde_lbc POSTHOOK: query: insert overwrite table date_serde_lbc - select c1, c2 from date_serde_c limit 1 + select c1, c2 from date_serde_c order by c1, c2 limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_c POSTHOOK: Output: default@date_serde_lbc @@ -453,7 +453,7 @@ POSTHOOK: query: select * from date_serde_lbc POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lbc #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: select c1, sum(c2) from date_serde_lbc group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_lbc @@ -462,7 +462,7 @@ POSTHOOK: query: select c1, sum(c2) from date_serde_lbc group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lbc #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: create temporary table date_serde_orc ( c1 date, c2 int @@ -486,12 +486,12 @@ POSTHOOK: type: ALTERTABLE_SERIALIZER POSTHOOK: Input: default@date_serde_orc POSTHOOK: Output: default@date_serde_orc PREHOOK: query: insert overwrite table date_serde_orc - select c1, c2 from date_serde_lbc limit 1 + select c1, c2 from date_serde_lbc order by c1, c2 limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_lbc PREHOOK: Output: default@date_serde_orc POSTHOOK: query: insert overwrite table date_serde_orc - select c1, c2 from date_serde_lbc limit 1 + select c1, c2 from date_serde_lbc order by c1, c2 limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_lbc POSTHOOK: Output: default@date_serde_orc @@ -505,7 +505,7 @@ POSTHOOK: query: select * from date_serde_orc POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_orc #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 PREHOOK: query: select c1, sum(c2) from date_serde_orc group by c1 PREHOOK: type: QUERY PREHOOK: Input: default@date_serde_orc @@ -514,4 +514,4 @@ POSTHOOK: query: select c1, sum(c2) from date_serde_orc group by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@date_serde_orc #### A masked pattern was here #### -2010-10-20 1064 +2010-10-20 361 diff --git ql/src/test/results/clientpositive/temp_table_parquet_mixed_partition_formats2.q.out ql/src/test/results/clientpositive/llap/temp_table_parquet_mixed_partition_formats2.q.out similarity index 90% rename from ql/src/test/results/clientpositive/temp_table_parquet_mixed_partition_formats2.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_parquet_mixed_partition_formats2.q.out index 23bb41edfe..d6c9f56535 100644 --- ql/src/test/results/clientpositive/temp_table_parquet_mixed_partition_formats2.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_parquet_mixed_partition_formats2.q.out @@ -37,12 +37,12 @@ POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@parquet_table_json_partition_temp POSTHOOK: Output: default@parquet_table_json_partition_temp@ts=20150101 -PREHOOK: query: SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +PREHOOK: query: SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_table_json_partition_temp PREHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +POSTHOOK: query: SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_table_json_partition_temp POSTHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 @@ -63,37 +63,37 @@ SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' POSTHOOK: type: ALTERTABLE_FILEFORMAT POSTHOOK: Input: default@parquet_table_json_partition_temp POSTHOOK: Output: default@parquet_table_json_partition_temp -PREHOOK: query: SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +PREHOOK: query: SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_table_json_partition_temp PREHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +POSTHOOK: query: SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_table_json_partition_temp POSTHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 #### A masked pattern was here #### 1 {"country":1,"state":1} [2,3] 20150101 2 {"country":1,"state":2} [] 20150101 -PREHOOK: query: CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +PREHOOK: query: CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@parquet_table_json_partition_temp PREHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 PREHOOK: Output: database:default PREHOOK: Output: default@new_table_temp -POSTHOOK: query: CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp LIMIT 100 +POSTHOOK: query: CREATE TEMPORARY TABLE new_table_temp AS SELECT * FROM parquet_table_json_partition_temp ORDER BY id, address, reports LIMIT 100 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@parquet_table_json_partition_temp POSTHOOK: Input: default@parquet_table_json_partition_temp@ts=20150101 POSTHOOK: Output: database:default POSTHOOK: Output: default@new_table_temp -PREHOOK: query: SELECT * FROM new_table_temp +PREHOOK: query: SELECT * FROM new_table_temp ORDER by id, address, reports PREHOOK: type: QUERY PREHOOK: Input: default@new_table_temp #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM new_table_temp +POSTHOOK: query: SELECT * FROM new_table_temp ORDER by id, address, reports POSTHOOK: type: QUERY POSTHOOK: Input: default@new_table_temp #### A masked pattern was here #### -2 {"country":1,"state":2} [] 20150101 1 {"country":1,"state":1} [2,3] 20150101 +2 {"country":1,"state":2} [] 20150101 diff --git ql/src/test/results/clientpositive/llap/temp_table_partition_boolexpr.q.out ql/src/test/results/clientpositive/llap/temp_table_partition_boolexpr.q.out new file mode 100644 index 0000000000..0f18cdaf45 --- /dev/null +++ ql/src/test/results/clientpositive/llap/temp_table_partition_boolexpr.q.out @@ -0,0 +1,317 @@ +PREHOOK: query: create temporary table part_boolexpr_temp(key int, value string) partitioned by (dt int, ts string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_boolexpr_temp +POSTHOOK: query: create temporary table part_boolexpr_temp(key int, value string) partitioned by (dt int, ts string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_boolexpr_temp +PREHOOK: query: select count(*) from part_boolexpr_temp where key = 'abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@part_boolexpr_temp +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from part_boolexpr_temp where key = 'abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_boolexpr_temp +#### A masked pattern was here #### +0 +PREHOOK: query: select * from part_boolexpr_temp where dt = 'abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@part_boolexpr_temp +#### A masked pattern was here #### +POSTHOOK: query: select * from part_boolexpr_temp where dt = 'abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_boolexpr_temp +#### A masked pattern was here #### +PREHOOK: query: explain select count(1) from srcpart where true +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where true +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where false +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where true and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where true and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where true or hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where true or hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false or hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where false or hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where false and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where INPUT__FILE__NAME is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: explain select count(1) from srcpart where INPUT__FILE__NAME is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: INPUT__FILE__NAME is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: INPUT__FILE__NAME is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/temp_table_partition_condition_remover.q.out ql/src/test/results/clientpositive/llap/temp_table_partition_condition_remover.q.out similarity index 94% rename from ql/src/test/results/clientpositive/temp_table_partition_condition_remover.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_partition_condition_remover.q.out index 18f5348f0f..a6b81bceb3 100644 --- ql/src/test/results/clientpositive/temp_table_partition_condition_remover.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_partition_condition_remover.q.out @@ -49,11 +49,9 @@ STAGE PLANS: TableScan alias: foo_n5_temp filterExpr: (s <> 'bar') (type: boolean) - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: i (type: int), s (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: select * from foo_n5_temp where s not in ('bar') diff --git ql/src/test/results/clientpositive/temp_table_partition_ctas.q.out ql/src/test/results/clientpositive/llap/temp_table_partition_ctas.q.out similarity index 95% rename from ql/src/test/results/clientpositive/temp_table_partition_ctas.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_partition_ctas.q.out index bd3574f03f..9a3b4ec01a 100644 --- ql/src/test/results/clientpositive/temp_table_partition_ctas.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_partition_ctas.q.out @@ -15,48 +15,61 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@partition_ctas_1_temp STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-3 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-3 - Stage-2 depends on stages: Stage-0, Stage-3 + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-0, Stage-4 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 300.0D)) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 300.0D)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 300.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 300.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Select Operator - expressions: value (type: string), key (type: string) + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), KEY._col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.partition_ctas_1_temp + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partition_ctas_1_temp - Stage: Stage-3 + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 Create Table columns: value string name: default.partition_ctas_1_temp @@ -79,7 +92,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.partition_ctas_1_temp - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: @@ -282,11 +295,9 @@ STAGE PLANS: TableScan alias: partition_ctas_1_temp filterExpr: (238.0D = 238.0D) (type: boolean) - Statistics: Num rows: 2 Data size: 550 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 550 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM partition_ctas_1_temp where key = 238 @@ -461,11 +472,9 @@ STAGE PLANS: TableScan alias: partition_ctas_2_temp filterExpr: (value = 'val_238') (type: boolean) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), 'val_238' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT * FROM partition_ctas_2_temp where value = 'val_238' @@ -615,38 +624,22 @@ POSTHOOK: Input: default@partition_ctas_2_temp@value=val_296 POSTHOOK: Input: default@partition_ctas_2_temp@value=val_298 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: partition_ctas_2_temp - filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) - Statistics: Num rows: 101 Data size: 36432 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: (UDFToDouble(key) = 238.0D) (type: boolean) - Statistics: Num rows: 50 Data size: 18216 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 50 Data size: 9200 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 50 Data size: 9200 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: partition_ctas_2_temp + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink PREHOOK: query: SELECT value FROM partition_ctas_2_temp where key = 238 PREHOOK: type: QUERY @@ -942,11 +935,9 @@ STAGE PLANS: TableScan alias: partition_ctas_diff_order_temp filterExpr: (value = 'val_238') (type: boolean) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), 'val_238' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT * FROM partition_ctas_diff_order_temp where value = 'val_238' @@ -1113,11 +1104,9 @@ STAGE PLANS: TableScan alias: partition_ctas_complex_order_temp filterExpr: (c0 = 'val_238_0') (type: boolean) - Statistics: Num rows: 2 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: c2 (type: string), c3 (type: string), c5 (type: string), 'val_238_0' (type: string), c4 (type: string), c1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 2026 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: SELECT * FROM partition_ctas_complex_order_temp where c0 = 'val_238_0' diff --git ql/src/test/results/clientpositive/temp_table_partition_multilevels.q.out ql/src/test/results/clientpositive/llap/temp_table_partition_multilevels.q.out similarity index 92% rename from ql/src/test/results/clientpositive/temp_table_partition_multilevels.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_partition_multilevels.q.out index 2ea8bf8631..1fcd595dab 100644 --- ql/src/test/results/clientpositive/temp_table_partition_multilevels.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_partition_multilevels.q.out @@ -6,7 +6,7 @@ POSTHOOK: query: create temporary table partition_test_multilevel_temp (key stri POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@partition_test_multilevel_temp -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -14,7 +14,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -24,7 +24,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -32,7 +32,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -42,7 +42,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -50,7 +50,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -60,7 +60,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -68,7 +68,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -78,7 +78,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=111/level3=44 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=44).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=111,level3=44).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -86,7 +86,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -96,7 +96,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -104,7 +104,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -114,7 +114,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -122,7 +122,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -132,7 +132,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -140,7 +140,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -150,7 +150,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=222/level3=44 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=44).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=222,level3=44).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -158,7 +158,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -168,7 +168,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -176,7 +176,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -186,7 +186,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -194,7 +194,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -204,7 +204,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -212,7 +212,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='1111', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -222,7 +222,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=1111/level2=333/level3=44 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=44).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=1111,level2=333,level3=44).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -230,7 +230,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='11') select key, value from srcpart tablesample (11 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -240,7 +240,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -248,7 +248,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='22') select key, value from srcpart tablesample (12 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -258,7 +258,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -266,7 +266,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='33') select key, value from srcpart tablesample (13 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -276,7 +276,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -284,7 +284,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='111', level3='44') select key, value from srcpart tablesample (14 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -294,7 +294,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=111/level3=44 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=44).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=111,level3=44).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -302,7 +302,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='11') select key, value from srcpart tablesample (15 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -312,7 +312,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -320,7 +320,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='22') select key, value from srcpart tablesample (16 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -330,7 +330,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -338,7 +338,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='33') select key, value from srcpart tablesample (17 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -348,7 +348,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -356,7 +356,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='222', level3='44') select key, value from srcpart tablesample (18 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -366,7 +366,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=222/level3=44 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=44).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=222,level3=44).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -374,7 +374,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=11 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='11') select key, value from srcpart tablesample (19 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -384,7 +384,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=11 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -392,7 +392,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=22 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='22') select key, value from srcpart tablesample (20 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -402,7 +402,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=22 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=22).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=22).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -410,7 +410,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=33 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='33') select key, value from srcpart tablesample (21 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -420,7 +420,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=33 POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=33).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_test_multilevel_temp PARTITION(level1=2222,level2=333,level3=33).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) +PREHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -428,7 +428,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@partition_test_multilevel_temp@level1=2222/level2=333/level3=44 -POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) +POSTHOOK: query: insert overwrite table partition_test_multilevel_temp partition(level1='2222', level2='333', level3='44') select key, value from srcpart tablesample (22 rows) order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 @@ -988,49 +988,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: partition_test_multilevel_temp - filterExpr: ((level1 = '2222') and level2 BETWEEN '222' AND '333' and level3 BETWEEN '11' AND '33') (type: boolean) - Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: level2 (type: string), level3 (type: string) - outputColumnNames: level2, level3 - Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: partition_test_multilevel_temp + filterExpr: ((level1 = '2222') and level2 BETWEEN '222' AND '333' and level3 BETWEEN '11' AND '33') (type: boolean) + Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: level2 (type: string), level3 (type: string) + outputColumnNames: level2, level3 + Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: level2 (type: string), level3 (type: string) + minReductionHashAggr: 0.962963 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: level2 (type: string), level3 (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1588,49 +1597,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: partition_test_multilevel_temp - filterExpr: ((level1 = '2222') and level2 BETWEEN '222' AND '333' and level3 BETWEEN '11' AND '33') (type: boolean) - Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: level2 (type: string), level3 (type: string) - outputColumnNames: level2, level3 - Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: partition_test_multilevel_temp + filterExpr: ((level1 = '2222') and level2 BETWEEN '222' AND '333' and level3 BETWEEN '11' AND '33') (type: boolean) + Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: level2 (type: string), level3 (type: string) + outputColumnNames: level2, level3 + Statistics: Num rows: 108 Data size: 40890 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: level2 (type: string), level3 (type: string) + minReductionHashAggr: 0.962963 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: level2 (type: string), level3 (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 2256 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/temp_table_partition_pruning.q.out ql/src/test/results/clientpositive/llap/temp_table_partition_pruning.q.out similarity index 73% rename from ql/src/test/results/clientpositive/temp_table_partition_pruning.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_partition_pruning.q.out index f6fdd61928..05556418de 100644 --- ql/src/test/results/clientpositive/temp_table_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_partition_pruning.q.out @@ -116,57 +116,14 @@ OPTIMIZED SQL: SELECT `customer`, `dt` FROM `default`.`daysales_temp` WHERE NVL(`dt` = '2001-01-01' AND `customer` = 1, FALSE) STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: daysales_temp - filterExpr: COALESCE(((dt = '2001-01-01') and (customer = 1)),false) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: COALESCE(((dt = '2001-01-01') and (customer = 1)),false) (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: customer (type: int), dt (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: dt=2001-01-01 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -215,9 +172,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp -#### A masked pattern was here #### Partition - base file name: dt=2001-01-03 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -266,14 +221,18 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp - Truncated Path -> Alias: -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: daysales_temp + filterExpr: COALESCE(((dt = '2001-01-01') and (customer = 1)),false) (type: boolean) + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: COALESCE(((dt = '2001-01-01') and (customer = 1)),false) (type: boolean) + Select Operator + expressions: customer (type: int), dt (type: string) + outputColumnNames: _col0, _col1 + ListSink PREHOOK: query: explain extended select * from daysales_temp where nvl(dt='2001-01-01' or customer=3, false) PREHOOK: type: QUERY @@ -291,57 +250,14 @@ OPTIMIZED SQL: SELECT `customer`, `dt` FROM `default`.`daysales_temp` WHERE NVL(`dt` = '2001-01-01' OR `customer` = 3, FALSE) STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: daysales_temp - filterExpr: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: customer (type: int), dt (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: dt=2001-01-01 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -390,9 +306,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp -#### A masked pattern was here #### Partition - base file name: dt=2001-01-03 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -441,14 +355,18 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp - Truncated Path -> Alias: -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: daysales_temp + filterExpr: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) + Select Operator + expressions: customer (type: int), dt (type: string) + outputColumnNames: _col0, _col1 + ListSink PREHOOK: query: explain extended select * from daysales_temp where nvl(dt='2001-01-01' or customer=3, false) PREHOOK: type: QUERY @@ -466,57 +384,14 @@ OPTIMIZED SQL: SELECT `customer`, `dt` FROM `default`.`daysales_temp` WHERE NVL(`dt` = '2001-01-01' OR `customer` = 3, FALSE) STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: daysales_temp - filterExpr: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: customer (type: int), dt (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: dt=2001-01-01 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -565,9 +440,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp -#### A masked pattern was here #### Partition - base file name: dt=2001-01-03 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -616,12 +489,16 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.daysales_temp name: default.daysales_temp - Truncated Path -> Alias: -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: daysales_temp + filterExpr: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: COALESCE(((dt = '2001-01-01') or (customer = 3)),false) (type: boolean) + Select Operator + expressions: customer (type: int), dt (type: string) + outputColumnNames: _col0, _col1 + ListSink diff --git ql/src/test/results/clientpositive/temp_table_windowing_expressions.q.out ql/src/test/results/clientpositive/llap/temp_table_windowing_expressions.q.out similarity index 100% rename from ql/src/test/results/clientpositive/temp_table_windowing_expressions.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_windowing_expressions.q.out index c45f36e988..96114e0389 100644 --- ql/src/test/results/clientpositive/temp_table_windowing_expressions.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_windowing_expressions.q.out @@ -571,28 +571,28 @@ POSTHOOK: Input: default@part Manufacturer#1 1753.76 Manufacturer#1 1632.66 Manufacturer#1 1602.59 -Manufacturer#1 1173.15 -Manufacturer#1 1173.15 Manufacturer#1 1414.42 Manufacturer#2 1800.7 Manufacturer#2 1690.68 Manufacturer#2 2031.98 -Manufacturer#2 1698.66 Manufacturer#2 1701.6 -Manufacturer#3 1922.98 Manufacturer#3 1410.39 Manufacturer#3 1671.68 Manufacturer#3 1190.27 Manufacturer#3 1337.29 -Manufacturer#4 1844.92 Manufacturer#4 1375.42 +Manufacturer#5 1788.73 +Manufacturer#1 1173.15 +Manufacturer#1 1173.15 +Manufacturer#2 1698.66 +Manufacturer#3 1922.98 +Manufacturer#4 1844.92 Manufacturer#4 1620.67 Manufacturer#4 1206.26 Manufacturer#4 1290.35 Manufacturer#5 1018.1 Manufacturer#5 1464.48 Manufacturer#5 1789.69 -Manufacturer#5 1788.73 Manufacturer#5 1611.66 PREHOOK: query: select p_mfgr, avg(p_retailprice) over(partition by p_mfgr order by p_type,p_mfgr rows between unbounded preceding and current row) from part PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/test_teradatabinaryfile.q.out ql/src/test/results/clientpositive/llap/test_teradatabinaryfile.q.out similarity index 100% rename from ql/src/test/results/clientpositive/test_teradatabinaryfile.q.out rename to ql/src/test/results/clientpositive/llap/test_teradatabinaryfile.q.out index 75584e9ba2..b0394f52e5 100644 --- ql/src/test/results/clientpositive/test_teradatabinaryfile.q.out +++ ql/src/test/results/clientpositive/llap/test_teradatabinaryfile.q.out @@ -392,12 +392,12 @@ POSTHOOK: Input: default@teradata_binary_table_1mb -127 3.14 -6 0.00 -4 3.14 --1 314.15 2 NULL -3 3140000000000.00 5 314000000.00 7 NULL 127 0.04 +-1 314.15 +3 3140000000000.00 NULL 12.00 PREHOOK: query: INSERT OVERWRITE TABLE teradata_binary_table_64kb_insert SELECT test_tinyint, test_decimal, test_date, test_timestamp FROM teradata_binary_table_64kb diff --git ql/src/test/results/clientpositive/llap/timestamp.q.out ql/src/test/results/clientpositive/llap/timestamp.q.out new file mode 100644 index 0000000000..e48227eb43 --- /dev/null +++ ql/src/test/results/clientpositive/llap/timestamp.q.out @@ -0,0 +1,447 @@ +PREHOOK: query: explain select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: boolean) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain extended select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain extended select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +OPTIMIZED SQL: SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `c` +FROM (SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `$f0` +FROM `default`.`src` +UNION ALL +SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `$f0` +FROM `default`.`src`) AS `t1` +GROUP BY TRUE +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: boolean) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: boolean) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: boolean) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0 + columns.types timestamp + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 01:01:01 +PREHOOK: query: explain select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: boolean) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: TIMESTAMP'2011-01-01 01:01:01.123' (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 01:01:01.123 diff --git ql/src/test/results/clientpositive/timestamp_comparison3.q.out ql/src/test/results/clientpositive/llap/timestamp_comparison3.q.out similarity index 55% rename from ql/src/test/results/clientpositive/timestamp_comparison3.q.out rename to ql/src/test/results/clientpositive/llap/timestamp_comparison3.q.out index 3977be77f7..c152a1069f 100644 --- ql/src/test/results/clientpositive/timestamp_comparison3.q.out +++ ql/src/test/results/clientpositive/llap/timestamp_comparison3.q.out @@ -70,27 +70,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: onecolumntable - filterExpr: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-04 00:00:00' (type: boolean) - Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-04 00:00:00' (type: boolean) - Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ts (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: onecolumntable + filterExpr: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-04 00:00:00' (type: boolean) + Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-04 00:00:00' (type: boolean) + Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ts (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -118,27 +122,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: onecolumntable - filterExpr: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-03 00:00:00' (type: boolean) - Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-03 00:00:00' (type: boolean) - Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ts (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: onecolumntable + filterExpr: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-03 00:00:00' (type: boolean) + Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ts BETWEEN TIMESTAMP'2015-01-02 00:00:00' AND TIMESTAMP'2015-01-03 00:00:00' (type: boolean) + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ts (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -166,27 +174,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: onecolumntable - filterExpr: ts BETWEEN TIMESTAMP'2015-01-01 00:00:00' AND TIMESTAMP'2015-01-08 00:00:00' (type: boolean) - Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ts BETWEEN TIMESTAMP'2015-01-01 00:00:00' AND TIMESTAMP'2015-01-08 00:00:00' (type: boolean) - Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ts (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: onecolumntable + filterExpr: ts BETWEEN TIMESTAMP'2015-01-01 00:00:00' AND TIMESTAMP'2015-01-08 00:00:00' (type: boolean) Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Filter Operator + predicate: ts BETWEEN TIMESTAMP'2015-01-01 00:00:00' AND TIMESTAMP'2015-01-08 00:00:00' (type: boolean) + Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ts (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/timestamp_ints_casts.q.out ql/src/test/results/clientpositive/llap/timestamp_ints_casts.q.out similarity index 78% rename from ql/src/test/results/clientpositive/timestamp_ints_casts.q.out rename to ql/src/test/results/clientpositive/llap/timestamp_ints_casts.q.out index 572c49ea72..8661344346 100644 --- ql/src/test/results/clientpositive/timestamp_ints_casts.q.out +++ ql/src/test/results/clientpositive/llap/timestamp_ints_casts.q.out @@ -39,38 +39,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ((cbigint % 250L) = 0L) (type: boolean) - Statistics: Num rows: 12288 Data size: 1559690 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((cbigint % 250L) = 0L) (type: boolean) - Statistics: Num rows: 6144 Data size: 779900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0L) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 6144 Data size: 2641080 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6144 Data size: 2641080 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: alltypesorc + filterExpr: ((cbigint % 250L) = 0L) (type: boolean) + Filter Operator + predicate: ((cbigint % 250L) = 0L) (type: boolean) + Select Operator + expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0L) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + ListSink PREHOOK: query: select @@ -177,38 +161,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ((cbigint % 250L) = 0L) (type: boolean) - Statistics: Num rows: 12288 Data size: 1559690 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((cbigint % 250L) = 0L) (type: boolean) - Statistics: Num rows: 6144 Data size: 779900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0L) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 6144 Data size: 2641080 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6144 Data size: 2641080 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: alltypesorc + filterExpr: ((cbigint % 250L) = 0L) (type: boolean) + Filter Operator + predicate: ((cbigint % 250L) = 0L) (type: boolean) + Select Operator + expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0L) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/timestamp_literal.q.out ql/src/test/results/clientpositive/llap/timestamp_literal.q.out similarity index 85% rename from ql/src/test/results/clientpositive/timestamp_literal.q.out rename to ql/src/test/results/clientpositive/llap/timestamp_literal.q.out index cfcd06f907..513009dda4 100644 --- ql/src/test/results/clientpositive/timestamp_literal.q.out +++ ql/src/test/results/clientpositive/llap/timestamp_literal.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select timestamp '2011-01-01 01:01:01' @@ -56,11 +54,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: true (type: boolean) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select '2011-01-01 01:01:01.101' <> timestamp '2011-01-01 01:01:01.100' @@ -93,11 +89,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select 1 where timestamp '2011-01-01 01:01:01.101' <> timestamp '2011-01-01 01:01:01.100' diff --git ql/src/test/results/clientpositive/timestamptz.q.out ql/src/test/results/clientpositive/llap/timestamptz.q.out similarity index 86% rename from ql/src/test/results/clientpositive/timestamptz.q.out rename to ql/src/test/results/clientpositive/llap/timestamptz.q.out index 09c50ddf10..1aeb3da727 100644 --- ql/src/test/results/clientpositive/timestamptz.q.out +++ ql/src/test/results/clientpositive/llap/timestamptz.q.out @@ -17,11 +17,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMPLOCALTZ'2005-01-02 18:01:00.0 US/Pacific' (type: timestamp with local time zone) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select cast('2005-01-03 02:01:00 GMT' as timestamp with local time zone) @@ -52,11 +50,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMPLOCALTZ'2016-01-03 12:26:34.0123 US/Pacific' (type: timestamp with local time zone) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select cast('2016-01-03 12:26:34.0123 America/Los_Angeles' as timestamplocaltz) @@ -87,11 +83,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMPLOCALTZ'2016-01-02 16:00:00.0 US/Pacific' (type: timestamp with local time zone) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select cast('2016-01-03Europe/London' as timestamplocaltz) @@ -122,11 +116,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMPLOCALTZ'2016-01-03 04:34:56.38 US/Pacific' (type: timestamp with local time zone) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select cast('2016-01-03 13:34:56.38 +1:00' as timestamplocaltz) diff --git ql/src/test/results/clientpositive/truncate_column_buckets.q.out ql/src/test/results/clientpositive/llap/truncate_column_buckets.q.out similarity index 100% rename from ql/src/test/results/clientpositive/truncate_column_buckets.q.out rename to ql/src/test/results/clientpositive/llap/truncate_column_buckets.q.out diff --git ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out ql/src/test/results/clientpositive/llap/truncate_column_list_bucket.q.out similarity index 63% rename from ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out rename to ql/src/test/results/clientpositive/llap/truncate_column_list_bucket.q.out index c8e40bd447..46a65e2b22 100644 --- ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ ql/src/test/results/clientpositive/llap/truncate_column_list_bucket.q.out @@ -63,57 +63,14 @@ OPTIMIZED SQL: SELECT CAST('484' AS STRING) AS `key`, `value`, CAST('1' AS STRIN FROM `default`.`test_tab_n3` WHERE `key` = '484' AND `part` = '1' STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_tab_n3 - filterExpr: ((key = '484') and (part = '1')) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key = '484') (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '484' (type: string), value (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 526 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 526 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: key=484 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -158,14 +115,18 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.test_tab_n3 name: default.test_tab_n3 - Truncated Path -> Alias: - /test_tab_n3/part=1/key=484 [test_tab_n3] - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: test_tab_n3 + filterExpr: ((key = '484') and (part = '1')) (type: boolean) + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key = '484') (type: boolean) + Select Operator + expressions: '484' (type: string), value (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink PREHOOK: query: SELECT * FROM test_tab_n3 WHERE part = '1' AND key = '484' PREHOOK: type: QUERY @@ -192,57 +153,14 @@ OPTIMIZED SQL: SELECT CAST('0' AS STRING) AS `key`, `value`, CAST('1' AS STRING) FROM `default`.`test_tab_n3` WHERE `key` = '0' AND `part` = '1' STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_tab_n3 - filterExpr: ((key = '0') and (part = '1')) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key = '0') (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '0' (type: string), value (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 522 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 522 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -287,14 +205,18 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.test_tab_n3 name: default.test_tab_n3 - Truncated Path -> Alias: - /test_tab_n3/part=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [test_tab_n3] - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: test_tab_n3 + filterExpr: ((key = '0') and (part = '1')) (type: boolean) + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key = '0') (type: boolean) + Select Operator + expressions: '0' (type: string), value (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink PREHOOK: query: SELECT * FROM test_tab_n3 WHERE part = '1' AND key = '0' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/type_cast_1.q.out ql/src/test/results/clientpositive/llap/type_cast_1.q.out similarity index 77% rename from ql/src/test/results/clientpositive/type_cast_1.q.out rename to ql/src/test/results/clientpositive/llap/type_cast_1.q.out index 22dad1a0f2..4d16dbcda5 100644 --- ql/src/test/results/clientpositive/type_cast_1.q.out +++ ql/src/test/results/clientpositive/llap/type_cast_1.q.out @@ -18,14 +18,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(false, 1, cast(2 as smallint)) + 3 FROM src LIMIT 1 diff --git ql/src/test/results/clientpositive/type_widening.q.out ql/src/test/results/clientpositive/llap/type_widening.q.out similarity index 83% rename from ql/src/test/results/clientpositive/type_widening.q.out rename to ql/src/test/results/clientpositive/llap/type_widening.q.out index f295e66ee9..360b356f53 100644 --- ql/src/test/results/clientpositive/type_widening.q.out +++ ql/src/test/results/clientpositive/llap/type_widening.q.out @@ -16,14 +16,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 0L (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT COALESCE(0, 9223372036854775807) FROM src LIMIT 1 @@ -49,48 +46,62 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 0L (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 9223372036854775807L (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 0L (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 9223372036854775807L (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + + File Output Operator + compressed: false Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1133,14 +1144,11 @@ STAGE PLANS: TableScan alias: t1_n114 filterExpr: (a > 2Y) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (a > 2Y) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: tinyint), b (type: smallint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select * from t1_n114 where b < 2 @@ -1162,14 +1170,11 @@ STAGE PLANS: TableScan alias: t1_n114 filterExpr: (b < 2S) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (b < 2S) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: tinyint), b (type: smallint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select * from t1_n114 where a < 200 @@ -1191,14 +1196,11 @@ STAGE PLANS: TableScan alias: t1_n114 filterExpr: (UDFToInteger(a) < 200) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToInteger(a) < 200) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: tinyint), b (type: smallint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select * from t1_n114 where b > 40000 @@ -1220,14 +1222,11 @@ STAGE PLANS: TableScan alias: t1_n114 filterExpr: (UDFToInteger(b) > 40000) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToInteger(b) > 40000) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: tinyint), b (type: smallint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: drop table t1_n114 diff --git ql/src/test/results/clientpositive/udaf_binarysetfunctions.q.out ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions.q.out similarity index 71% rename from ql/src/test/results/clientpositive/udaf_binarysetfunctions.q.out rename to ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions.q.out index 86dbcf6f57..81c56c28e7 100644 --- ql/src/test/results/clientpositive/udaf_binarysetfunctions.q.out +++ ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions.q.out @@ -372,75 +372,75 @@ POSTHOOK: Input: default@t_n21 #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t_n21 - Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: px (type: int), x (type: decimal(10,0)), y (type: decimal(10,0)), UDFToDouble(x) (type: double), (UDFToDouble(x) * UDFToDouble(x)) (type: double), UDFToDouble(y) (type: double), (UDFToDouble(y) * UDFToDouble(y)) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t_n21 + Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: px (type: int), x (type: decimal(10,0)), y (type: decimal(10,0)), UDFToDouble(x) (type: double), (UDFToDouble(x) * UDFToDouble(x)) (type: double), UDFToDouble(y) (type: double), (UDFToDouble(y) * UDFToDouble(y)) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col4), sum(_col3), count(_col1), sum(_col6), sum(_col5), count(_col2), corr(_col2, _col1), covar_samp(_col2, _col1), covar_pop(_col2, _col1), regr_count(_col2, _col1), regr_slope(_col2, _col1), regr_intercept(_col2, _col1), regr_r2(_col2, _col1), regr_sxx(_col2, _col1), regr_syy(_col2, _col1), regr_sxy(_col2, _col1), regr_avgx(_col2, _col1), regr_avgy(_col2, _col1) + keys: _col0 (type: int) + minReductionHashAggr: 0.7586207 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 + Statistics: Num rows: 7 Data size: 11396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 11396 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: bigint), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: bigint), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: struct), _col16 (type: struct), _col17 (type: struct), _col18 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: sum(_col4), sum(_col3), count(_col1), sum(_col6), sum(_col5), count(_col2), corr(_col2, _col1), covar_samp(_col2, _col1), covar_pop(_col2, _col1), regr_count(_col2, _col1), regr_slope(_col2, _col1), regr_intercept(_col2, _col1), regr_r2(_col2, _col1), regr_sxx(_col2, _col1), regr_syy(_col2, _col1), regr_sxy(_col2, _col1), regr_avgx(_col2, _col1), regr_avgy(_col2, _col1) - keys: _col0 (type: int) - minReductionHashAggr: 0.99 - mode: hash + aggregations: sum(VALUE._col0), sum(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), count(VALUE._col5), corr(VALUE._col6), covar_samp(VALUE._col7), covar_pop(VALUE._col8), regr_count(VALUE._col9), regr_slope(VALUE._col10), regr_intercept(VALUE._col11), regr_r2(VALUE._col12), regr_sxx(VALUE._col13), regr_syy(VALUE._col14), regr_sxy(VALUE._col15), regr_avgx(VALUE._col16), regr_avgy(VALUE._col17) + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 7 Data size: 11396 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 11396 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: bigint), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: bigint), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: struct), _col16 (type: struct), _col17 (type: struct), _col18 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), count(VALUE._col5), corr(VALUE._col6), covar_samp(VALUE._col7), covar_pop(VALUE._col8), regr_count(VALUE._col9), regr_slope(VALUE._col10), regr_intercept(VALUE._col11), regr_r2(VALUE._col12), regr_sxx(VALUE._col13), regr_syy(VALUE._col14), regr_sxy(VALUE._col15), regr_avgx(VALUE._col16), regr_avgy(VALUE._col17) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 7 Data size: 2492 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), ((_col1 - ((_col2 * _col2) / _col3)) / _col3) (type: double), ((_col4 - ((_col5 * _col5) / _col6)) / _col6) (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: bigint), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: decimal(14,4)), _col18 (type: decimal(14,4)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(14,4)), _col14 (type: decimal(14,4)) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: bigint), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: decimal(14,4)), VALUE._col13 (type: decimal(14,4)), VALUE._col5 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 7 Data size: 2492 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), ((_col1 - ((_col2 * _col2) / _col3)) / _col3) (type: double), ((_col4 - ((_col5 * _col5) / _col6)) / _col6) (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: bigint), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: decimal(14,4)), _col18 (type: decimal(14,4)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(14,4)), _col14 (type: decimal(14,4)) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: bigint), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: decimal(14,4)), VALUE._col13 (type: decimal(14,4)), VALUE._col5 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/udaf_binarysetfunctions_no_cbo.q.out ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions_no_cbo.q.out similarity index 76% rename from ql/src/test/results/clientpositive/udaf_binarysetfunctions_no_cbo.q.out rename to ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions_no_cbo.q.out index 6857ca9739..c53a5eca24 100644 --- ql/src/test/results/clientpositive/udaf_binarysetfunctions_no_cbo.q.out +++ ql/src/test/results/clientpositive/llap/udaf_binarysetfunctions_no_cbo.q.out @@ -372,71 +372,71 @@ POSTHOOK: Input: default@t_n6 #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t_n6 - Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: px (type: int), y (type: decimal(10,0)), x (type: decimal(10,0)) - outputColumnNames: px, y, x - Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t_n6 + Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: px (type: int), y (type: decimal(10,0)), x (type: decimal(10,0)) + outputColumnNames: px, y, x + Statistics: Num rows: 29 Data size: 5044 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: var_pop(x), var_pop(y), corr(y, x), covar_samp(y, x), covar_pop(y, x), regr_count(y, x), regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), regr_sxx(y, x), regr_syy(y, x), regr_sxy(y, x), regr_avgx(y, x), regr_avgy(y, x) + keys: px (type: int) + minReductionHashAggr: 0.7586207 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Statistics: Num rows: 7 Data size: 12180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 12180 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: bigint), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: var_pop(x), var_pop(y), corr(y, x), covar_samp(y, x), covar_pop(y, x), regr_count(y, x), regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), regr_sxx(y, x), regr_syy(y, x), regr_sxy(y, x), regr_avgx(y, x), regr_avgy(y, x) - keys: px (type: int) - minReductionHashAggr: 0.99 - mode: hash + aggregations: var_pop(VALUE._col0), var_pop(VALUE._col1), corr(VALUE._col2), covar_samp(VALUE._col3), covar_pop(VALUE._col4), regr_count(VALUE._col5), regr_slope(VALUE._col6), regr_intercept(VALUE._col7), regr_r2(VALUE._col8), regr_sxx(VALUE._col9), regr_syy(VALUE._col10), regr_sxy(VALUE._col11), regr_avgx(VALUE._col12), regr_avgy(VALUE._col13) + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 7 Data size: 12180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 2268 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: z sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 12180 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: bigint), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: var_pop(VALUE._col0), var_pop(VALUE._col1), corr(VALUE._col2), covar_samp(VALUE._col3), covar_pop(VALUE._col4), regr_count(VALUE._col5), regr_slope(VALUE._col6), regr_intercept(VALUE._col7), regr_r2(VALUE._col8), regr_sxx(VALUE._col9), regr_syy(VALUE._col10), regr_sxy(VALUE._col11), regr_avgx(VALUE._col12), regr_avgy(VALUE._col13) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Statistics: Num rows: 7 Data size: 2268 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 7 Data size: 2268 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(14,4)), _col14 (type: decimal(14,4)) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: bigint), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: decimal(14,4)), VALUE._col13 (type: decimal(14,4)), VALUE._col5 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 7 Data size: 2268 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: bigint), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(14,4)), _col14 (type: decimal(14,4)) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: bigint), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: decimal(14,4)), VALUE._col13 (type: decimal(14,4)), VALUE._col5 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 2324 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/udaf_number_format.q.out ql/src/test/results/clientpositive/llap/udaf_number_format.q.out new file mode 100644 index 0000000000..a1ae4cc77c --- /dev/null +++ ql/src/test/results/clientpositive/llap/udaf_number_format.q.out @@ -0,0 +1,95 @@ +PREHOOK: query: EXPLAIN SELECT + sum('a'), + avg('a'), + variance('a'), + std('a') +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT + sum('a'), + avg('a'), + variance('a'), + std('a') +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum('a'), count(), sum(null), sum(null) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: double), _col1 (type: bigint), _col2 (type: double), _col3 (type: double) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: double), (_col0 / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / _col1) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT + sum('a'), + avg('a'), + variance('a'), + std('a') +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT + sum('a'), + avg('a'), + variance('a'), + std('a') +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0.0 0.0 NULL NULL diff --git ql/src/test/results/clientpositive/udaf_percentile_cont.q.out ql/src/test/results/clientpositive/llap/udaf_percentile_cont.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udaf_percentile_cont.q.out rename to ql/src/test/results/clientpositive/llap/udaf_percentile_cont.q.out index 509ae7bfe6..8810a4cac0 100644 --- ql/src/test/results/clientpositive/udaf_percentile_cont.q.out +++ ql/src/test/results/clientpositive/llap/udaf_percentile_cont.q.out @@ -498,43 +498,53 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t_test - Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t_test + Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: percentile_cont(_col0, 0), percentile_cont(_col0, 0.2), percentile_cont(0.2, _col0, 1, 0), percentile_cont(0.2, _col0, 1, 1), percentile_cont(0.2, _col0, 0, 0), percentile_cont(0.2, _col0, 0, 1) + minReductionHashAggr: 0.9411765 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct,percentiles:array,isascending:boolean>), _col1 (type: struct,percentiles:array,isascending:boolean>), _col2 (type: struct,percentiles:array,isascending:boolean>), _col3 (type: struct,percentiles:array,isascending:boolean>), _col4 (type: struct,percentiles:array,isascending:boolean>), _col5 (type: struct,percentiles:array,isascending:boolean>) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: percentile_cont(_col0, 0), percentile_cont(_col0, 0.2), percentile_cont(0.2, _col0, 1, 0), percentile_cont(0.2, _col0, 1, 1), percentile_cont(0.2, _col0, 0, 0), percentile_cont(0.2, _col0, 0, 1) - minReductionHashAggr: 0.99 - mode: hash + aggregations: percentile_cont(VALUE._col0), percentile_cont(VALUE._col1), percentile_cont(VALUE._col2), percentile_cont(VALUE._col3), percentile_cont(VALUE._col4), percentile_cont(VALUE._col5) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct,percentiles:array,isascending:boolean>), _col1 (type: struct,percentiles:array,isascending:boolean>), _col2 (type: struct,percentiles:array,isascending:boolean>), _col3 (type: struct,percentiles:array,isascending:boolean>), _col4 (type: struct,percentiles:array,isascending:boolean>), _col5 (type: struct,percentiles:array,isascending:boolean>) - Reduce Operator Tree: - Group By Operator - aggregations: percentile_cont(VALUE._col0), percentile_cont(VALUE._col1), percentile_cont(VALUE._col2), percentile_cont(VALUE._col3), percentile_cont(VALUE._col4), percentile_cont(VALUE._col5) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col2 (type: double), (_col2 = _col1) (type: boolean), _col2 (type: double), (_col2 = _col1) (type: boolean), _col4 (type: double), _col5 (type: double), _col4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col2 (type: double), (_col2 = _col1) (type: boolean), _col2 (type: double), (_col2 = _col1) (type: boolean), _col4 (type: double), _col5 (type: double), _col4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/udaf_percentile_disc.q.out ql/src/test/results/clientpositive/llap/udaf_percentile_disc.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udaf_percentile_disc.q.out rename to ql/src/test/results/clientpositive/llap/udaf_percentile_disc.q.out index e7efcf9302..ddb46ac5b4 100644 --- ql/src/test/results/clientpositive/udaf_percentile_disc.q.out +++ ql/src/test/results/clientpositive/llap/udaf_percentile_disc.q.out @@ -498,43 +498,53 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t_test - Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t_test + Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 17 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: percentile_disc(_col0, 0), percentile_disc(_col0, 0.2), percentile_disc(0.2, _col0, 1, 0), percentile_disc(0.2, _col0, 1, 1), percentile_disc(0.2, _col0, 0, 0), percentile_disc(0.2, _col0, 0, 1) + minReductionHashAggr: 0.9411765 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct,percentiles:array,isascending:boolean>), _col1 (type: struct,percentiles:array,isascending:boolean>), _col2 (type: struct,percentiles:array,isascending:boolean>), _col3 (type: struct,percentiles:array,isascending:boolean>), _col4 (type: struct,percentiles:array,isascending:boolean>), _col5 (type: struct,percentiles:array,isascending:boolean>) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: percentile_disc(_col0, 0), percentile_disc(_col0, 0.2), percentile_disc(0.2, _col0, 1, 0), percentile_disc(0.2, _col0, 1, 1), percentile_disc(0.2, _col0, 0, 0), percentile_disc(0.2, _col0, 0, 1) - minReductionHashAggr: 0.99 - mode: hash + aggregations: percentile_disc(VALUE._col0), percentile_disc(VALUE._col1), percentile_disc(VALUE._col2), percentile_disc(VALUE._col3), percentile_disc(VALUE._col4), percentile_disc(VALUE._col5) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 4728 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct,percentiles:array,isascending:boolean>), _col1 (type: struct,percentiles:array,isascending:boolean>), _col2 (type: struct,percentiles:array,isascending:boolean>), _col3 (type: struct,percentiles:array,isascending:boolean>), _col4 (type: struct,percentiles:array,isascending:boolean>), _col5 (type: struct,percentiles:array,isascending:boolean>) - Reduce Operator Tree: - Group By Operator - aggregations: percentile_disc(VALUE._col0), percentile_disc(VALUE._col1), percentile_disc(VALUE._col2), percentile_disc(VALUE._col3), percentile_disc(VALUE._col4), percentile_disc(VALUE._col5) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col2 (type: double), (_col2 = _col1) (type: boolean), _col2 (type: double), (_col2 = _col1) (type: boolean), _col4 (type: double), _col5 (type: double), _col4 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col2 (type: double), (_col2 = _col1) (type: boolean), _col2 (type: double), (_col2 = _col1) (type: boolean), _col4 (type: double), _col5 (type: double), _col4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/udf1.q.out ql/src/test/results/clientpositive/llap/udf1.q.out new file mode 100644 index 0000000000..05e23f0068 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf1.q.out @@ -0,0 +1,176 @@ +PREHOOK: query: CREATE TABLE dest1_n1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, + c5 STRING, c6 STRING, c7 STRING, c8 STRING, + c9 STRING, c10 STRING, c11 STRING, c12 STRING, c13 STRING, + c14 STRING, c15 STRING, c16 STRING, c17 STRING, + c18 STRING, c19 STRING, c20 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n1 +POSTHOOK: query: CREATE TABLE dest1_n1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, + c5 STRING, c6 STRING, c7 STRING, c8 STRING, + c9 STRING, c10 STRING, c11 STRING, c12 STRING, c13 STRING, + c14 STRING, c15 STRING, c16 STRING, c17 STRING, + c18 STRING, c19 STRING, c20 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n1 +PREHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', + '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', + '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', + REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), + REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), + REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' + WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n1 +POSTHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', + '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', + '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', + REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), + REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), + REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' + WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'TRUE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'FALSE' (type: string), 'FALSE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'acc' (type: string), 'abc' (type: string), 'abb' (type: string), 'hive' (type: string), 'hadoop' (type: string), 'AaAbAcA' (type: string), 'FALSE' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 + Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: string) + outputColumnNames: c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20 + Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll'), compute_stats(c8, 'hll'), compute_stats(c9, 'hll'), compute_stats(c10, 'hll'), compute_stats(c11, 'hll'), compute_stats(c12, 'hll'), compute_stats(c13, 'hll'), compute_stats(c14, 'hll'), compute_stats(c15, 'hll'), compute_stats(c16, 'hll'), compute_stats(c17, 'hll'), compute_stats(c18, 'hll'), compute_stats(c19, 'hll'), compute_stats(c20, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: struct), _col16 (type: struct), _col17 (type: struct), _col18 (type: struct), _col19 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5), compute_stats(VALUE._col6), compute_stats(VALUE._col7), compute_stats(VALUE._col8), compute_stats(VALUE._col9), compute_stats(VALUE._col10), compute_stats(VALUE._col11), compute_stats(VALUE._col12), compute_stats(VALUE._col13), compute_stats(VALUE._col14), compute_stats(VALUE._col15), compute_stats(VALUE._col16), compute_stats(VALUE._col17), compute_stats(VALUE._col18), compute_stats(VALUE._col19) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20 + Column Types: string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string + Table: default.dest1_n1 + +PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', + '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', + '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', + REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), + REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), + REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' + WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n1 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', + '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', + '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', + REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), + REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), + REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' + WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n1 +POSTHOOK: Lineage: dest1_n1.c1 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c10 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c11 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c12 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c13 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c14 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c15 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c16 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c17 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c18 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c19 SIMPLE [] +POSTHOOK: Lineage: dest1_n1.c2 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c20 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c3 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c4 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c5 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c6 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c7 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c8 EXPRESSION [] +POSTHOOK: Lineage: dest1_n1.c9 EXPRESSION [] +PREHOOK: query: SELECT dest1_n1.* FROM dest1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1_n1.* FROM dest1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n1 +#### A masked pattern was here #### +TRUE FALSE TRUE TRUE TRUE FALSE FALSE FALSE TRUE TRUE FALSE TRUE TRUE acc abc abb hive hadoop AaAbAcA FALSE diff --git ql/src/test/results/clientpositive/udf2.q.out ql/src/test/results/clientpositive/llap/udf2.q.out similarity index 60% rename from ql/src/test/results/clientpositive/udf2.q.out rename to ql/src/test/results/clientpositive/llap/udf2.q.out index bcc2faa16a..be6a35f443 100644 --- ql/src/test/results/clientpositive/udf2.q.out +++ ql/src/test/results/clientpositive/llap/udf2.q.out @@ -26,34 +26,19 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n55 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dest1_n55 - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '|' (type: string), trim(c1) (type: string), '|' (type: string), rtrim(c1) (type: string), '|' (type: string), ltrim(c1) (type: string), '|' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: dest1_n55 + Select Operator + expressions: '|' (type: string), trim(c1) (type: string), '|' (type: string), rtrim(c1) (type: string), '|' (type: string), ltrim(c1) (type: string), '|' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + ListSink PREHOOK: query: SELECT '|', trim(dest1_n55.c1), '|', rtrim(dest1_n55.c1), '|', ltrim(dest1_n55.c1), '|' FROM dest1_n55 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/udf3.q.out ql/src/test/results/clientpositive/llap/udf3.q.out new file mode 100644 index 0000000000..a3c993b781 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf3.q.out @@ -0,0 +1,136 @@ +PREHOOK: query: CREATE TABLE dest1_n104(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n104 +POSTHOOK: query: CREATE TABLE dest1_n104(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n104 +PREHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), +min(CAST('' AS INT)), max(CAST('' AS INT)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n104 +POSTHOOK: query: EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), +min(CAST('' AS INT)), max(CAST('' AS INT)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n104 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(null), sum(null), min(null), max(null) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: int), _col3 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1), min(VALUE._col2), max(VALUE._col3) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: CAST( _col0 AS STRING) (type: string), CAST( _col1 AS STRING) (type: string), CAST( (UDFToDouble(_col1) / _col0) AS STRING) (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n104 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: c1, c2, c3, c4, c5 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll') + mode: complete + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n104 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, c2, c3, c4, c5 + Column Types: string, string, string, string, string + Table: default.dest1_n104 + +PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), +min(CAST('' AS INT)), max(CAST('' AS INT)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n104 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), +min(CAST('' AS INT)), max(CAST('' AS INT)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n104 +POSTHOOK: Lineage: dest1_n104.c1 EXPRESSION [] +POSTHOOK: Lineage: dest1_n104.c2 EXPRESSION [] +POSTHOOK: Lineage: dest1_n104.c3 EXPRESSION [] +POSTHOOK: Lineage: dest1_n104.c4 EXPRESSION [] +POSTHOOK: Lineage: dest1_n104.c5 EXPRESSION [] +PREHOOK: query: SELECT dest1_n104.* FROM dest1_n104 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n104 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1_n104.* FROM dest1_n104 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n104 +#### A masked pattern was here #### +0 NULL NULL NULL NULL diff --git ql/src/test/results/clientpositive/udf4.q.out ql/src/test/results/clientpositive/llap/udf4.q.out similarity index 68% rename from ql/src/test/results/clientpositive/udf4.q.out rename to ql/src/test/results/clientpositive/llap/udf4.q.out index d9b841aab9..8d3a86fe56 100644 --- ql/src/test/results/clientpositive/udf4.q.out +++ ql/src/test/results/clientpositive/llap/udf4.q.out @@ -70,34 +70,19 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n149 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dest1_n149 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: decimal(1,0)), 2 (type: decimal(2,0)), -2 (type: decimal(2,0)), 1 (type: decimal(2,0)), 1 (type: decimal(2,0)), -2 (type: decimal(2,0)), 1.0D (type: double), null (type: double), 0.0D (type: double), 1 (type: decimal(2,0)), 2 (type: decimal(2,0)), -1 (type: decimal(2,0)), 1 (type: decimal(2,0)), rand(3) (type: double), 3 (type: int), -3 (type: int), 3 (type: int), -1 (type: int), -2 (type: int), -2Y (type: tinyint), -2S (type: smallint), -2L (type: bigint), 0Y (type: tinyint), 0S (type: smallint), 0 (type: int), 0L (type: bigint), 3Y (type: tinyint), 3S (type: smallint), 3 (type: int), 3L (type: bigint), 2Y (type: tinyint), 2S (type: smallint), 2 (type: int), 2L (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33 - Statistics: Num rows: 1 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: dest1_n149 + Select Operator + expressions: 1 (type: decimal(1,0)), 2 (type: decimal(2,0)), -2 (type: decimal(2,0)), 1 (type: decimal(2,0)), 1 (type: decimal(2,0)), -2 (type: decimal(2,0)), 1.0D (type: double), null (type: double), 0.0D (type: double), 1 (type: decimal(2,0)), 2 (type: decimal(2,0)), -1 (type: decimal(2,0)), 1 (type: decimal(2,0)), rand(3) (type: double), 3 (type: int), -3 (type: int), 3 (type: int), -1 (type: int), -2 (type: int), -2Y (type: tinyint), -2S (type: smallint), -2L (type: bigint), 0Y (type: tinyint), 0S (type: smallint), 0 (type: int), 0L (type: bigint), 3Y (type: tinyint), 3S (type: smallint), 3 (type: int), 3L (type: bigint), 2Y (type: tinyint), 2S (type: smallint), 2 (type: int), 2L (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33 + ListSink PREHOOK: query: SELECT round(1.0), round(1.5), round(-1.5), floor(1.0), floor(1.5), floor(-1.5), sqrt(1.0), sqrt(-1.0), sqrt(0.0), ceil(1.0), ceil(1.5), ceil(-1.5), ceiling(1.0), rand(3), +3, -3, 1++2, 1+-2, ~1, diff --git ql/src/test/results/clientpositive/udf5.q.out ql/src/test/results/clientpositive/llap/udf5.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf5.q.out rename to ql/src/test/results/clientpositive/llap/udf5.q.out index 58a1dab60b..21ee1b6be9 100644 --- ql/src/test/results/clientpositive/udf5.q.out +++ ql/src/test/results/clientpositive/llap/udf5.q.out @@ -35,11 +35,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n14 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2008-11-11 15:32:20' (type: string), DATE'2008-11-11' (type: date), 1 (type: int), 11 (type: int), 2008 (type: int), 1 (type: int), 11 (type: int), 2008 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 183 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT from_unixtime(1226446340), to_date(from_unixtime(1226446340)), day('2008-11-01'), month('2008-11-01'), year('2008-11-01'), day('2008-11-01 15:32:20'), month('2008-11-01 15:32:20'), year('2008-11-01 15:32:20') FROM dest1_n14 @@ -71,11 +69,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n14 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '01/13/10 11:57:40' (type: string), '2010-01-13 11:57:40' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT from_unixtime(unix_timestamp('2010-01-13 11:57:40', 'yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss'), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) from dest1_n14 diff --git ql/src/test/results/clientpositive/udf6.q.out ql/src/test/results/clientpositive/llap/udf6.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf6.q.out rename to ql/src/test/results/clientpositive/llap/udf6.q.out index e6d58324c7..81d429e68f 100644 --- ql/src/test/results/clientpositive/udf6.q.out +++ ql/src/test/results/clientpositive/llap/udf6.q.out @@ -35,11 +35,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n60 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(TRUE, 1, 2) FROM dest1_n60 @@ -81,11 +79,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n60 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 2 (type: int), 2 (type: int), 'a' (type: string), 0.1 (type: decimal(1,1)), 2L (type: bigint), 126Y (type: tinyint), 128S (type: smallint), 128 (type: int), 1.0D (type: double), '128' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(TRUE, 1, 2), IF(FALSE, 1, 2), IF(NULL, 1, 2), IF(TRUE, "a", "b"), diff --git ql/src/test/results/clientpositive/udf7.q.out ql/src/test/results/clientpositive/llap/udf7.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf7.q.out rename to ql/src/test/results/clientpositive/llap/udf7.q.out index 44b282f82e..b9e91ab3d2 100644 --- ql/src/test/results/clientpositive/udf7.q.out +++ ql/src/test/results/clientpositive/llap/udf7.q.out @@ -49,11 +49,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n111 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1.098612288668D (type: double), null (type: double), null (type: double), 1.098612288668D (type: double), null (type: double), null (type: double), 1.584962500721D (type: double), null (type: double), null (type: double), 0.47712125472D (type: double), null (type: double), null (type: double), 1.584962500721D (type: double), null (type: double), null (type: double), null (type: double), -1.0D (type: double), 7.389056098931D (type: double), 8.0D (type: double), 8.0D (type: double), 0.125D (type: double), 8.0D (type: double), 2.0D (type: double), NaND (type: double), 1.0D (type: double), 1.0D (type: double), 8.0D (type: double), 8.0D (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 - Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), diff --git ql/src/test/results/clientpositive/llap/udf8.q.out ql/src/test/results/clientpositive/llap/udf8.q.out new file mode 100644 index 0000000000..db9dd1f0b5 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf8.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: CREATE TABLE dest1_n54(c1 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n54 +POSTHOOK: query: CREATE TABLE dest1_n54(c1 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n54 +PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '' WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n54 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '' WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n54 +POSTHOOK: Lineage: dest1_n54.c1 SIMPLE [] +PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '1' WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n54 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '1' WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n54 +POSTHOOK: Lineage: dest1_n54.c1 SIMPLE [] +PREHOOK: query: EXPLAIN +SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n54 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n54 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dest1_n54 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: string) + outputColumnNames: c1 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(c1), count(c1) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: double), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (_col0 / _col1) (type: double), _col0 (type: double), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n54 +#### A masked pattern was here #### +POSTHOOK: query: SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n54 +#### A masked pattern was here #### +1.0 1.0 1 diff --git ql/src/test/results/clientpositive/udf9.q.out ql/src/test/results/clientpositive/llap/udf9.q.out similarity index 63% rename from ql/src/test/results/clientpositive/udf9.q.out rename to ql/src/test/results/clientpositive/llap/udf9.q.out index a55b3cdb34..7d1d4f7d9c 100644 --- ql/src/test/results/clientpositive/udf9.q.out +++ ql/src/test/results/clientpositive/llap/udf9.q.out @@ -25,38 +25,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: -1 (type: int), 2 (type: int), 32 (type: int), -1 (type: int), DATE'2009-01-01' (type: date), DATE'2009-12-31' (type: date), DATE'2008-03-01' (type: date), DATE'2009-03-02' (type: date), DATE'2008-02-28' (type: date), DATE'2009-02-27' (type: date), DATE'2008-12-31' (type: date), DATE'2008-01-02' (type: date), DATE'2008-02-26' (type: date), DATE'2009-02-26' (type: date), DATE'2006-02-28' (type: date), DATE'2005-02-28' (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 250 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Select Operator + expressions: -1 (type: int), 2 (type: int), 32 (type: int), -1 (type: int), DATE'2009-01-01' (type: date), DATE'2009-12-31' (type: date), DATE'2008-03-01' (type: date), DATE'2009-03-02' (type: date), DATE'2008-02-28' (type: date), DATE'2009-02-27' (type: date), DATE'2008-12-31' (type: date), DATE'2008-01-02' (type: date), DATE'2008-02-26' (type: date), DATE'2009-02-26' (type: date), DATE'2006-02-28' (type: date), DATE'2005-02-28' (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 + ListSink PREHOOK: query: SELECT DATEDIFF('2008-12-31', '2009-01-01'), DATEDIFF('2008-03-01', '2008-02-28'), DATEDIFF('2007-03-01', '2007-01-28'), DATEDIFF('2008-03-01 23:59:59', '2008-03-02 00:00:00'), diff --git ql/src/test/results/clientpositive/llap/udf_10_trims.q.out ql/src/test/results/clientpositive/llap/udf_10_trims.q.out new file mode 100644 index 0000000000..ddf22640a1 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf_10_trims.q.out @@ -0,0 +1,128 @@ +PREHOOK: query: CREATE TABLE dest1_n5(c1 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n5 +POSTHOOK: query: CREATE TABLE dest1_n5(c1 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n5 +PREHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE dest1_n5 +SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) +FROM src +WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n5 +POSTHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE dest1_n5 +SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) +FROM src +WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n5 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'abc' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n5 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: c1 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n5 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1 + Column Types: string + Table: default.dest1_n5 + +PREHOOK: query: INSERT OVERWRITE TABLE dest1_n5 +SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) +FROM src +WHERE src.key = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n5 +POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n5 +SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) +FROM src +WHERE src.key = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n5 +POSTHOOK: Lineage: dest1_n5.c1 SIMPLE [] diff --git ql/src/test/results/clientpositive/udf_E.q.out ql/src/test/results/clientpositive/llap/udf_E.q.out similarity index 87% rename from ql/src/test/results/clientpositive/udf_E.q.out rename to ql/src/test/results/clientpositive/llap/udf_E.q.out index 469f396a85..8a67e1dade 100644 --- ql/src/test/results/clientpositive/udf_E.q.out +++ ql/src/test/results/clientpositive/llap/udf_E.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2.718281828459045D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select E() FROM src tablesample (1 rows) @@ -72,11 +70,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2.718281828459045D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select E() FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_PI.q.out ql/src/test/results/clientpositive/llap/udf_PI.q.out similarity index 87% rename from ql/src/test/results/clientpositive/udf_PI.q.out rename to ql/src/test/results/clientpositive/llap/udf_PI.q.out index a9ec8c1e06..f7b8f7cd85 100644 --- ql/src/test/results/clientpositive/udf_PI.q.out +++ ql/src/test/results/clientpositive/llap/udf_PI.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 3.141592653589793D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select PI() FROM src tablesample (1 rows) @@ -72,11 +70,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 3.141592653589793D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select PI() FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_abs.q.out ql/src/test/results/clientpositive/llap/udf_abs.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udf_abs.q.out rename to ql/src/test/results/clientpositive/llap/udf_abs.q.out index fee7592ec9..9ba23cf3a7 100644 --- ql/src/test/results/clientpositive/udf_abs.q.out +++ ql/src/test/results/clientpositive/llap/udf_abs.q.out @@ -46,11 +46,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 0 (type: int), 1 (type: int), 123 (type: int), 9223372036854775807L (type: bigint), 9223372036854775807L (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 14000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -101,11 +99,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 0 (type: decimal(1,0)), 3.14159265 (type: decimal(9,8)), 3.14159265 (type: decimal(9,8)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 168000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_add_months.q.out ql/src/test/results/clientpositive/llap/udf_add_months.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_add_months.q.out rename to ql/src/test/results/clientpositive/llap/udf_add_months.q.out index 7150f59d68..61f9e07fad 100644 --- ql/src/test/results/clientpositive/udf_add_months.q.out +++ ql/src/test/results/clientpositive/llap/udf_add_months.q.out @@ -37,11 +37,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-02-14' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_aes_decrypt.q.out ql/src/test/results/clientpositive/llap/udf_aes_decrypt.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_aes_decrypt.q.out rename to ql/src/test/results/clientpositive/llap/udf_aes_decrypt.q.out index 89caf994cf..c4d983c82e 100644 --- ql/src/test/results/clientpositive/udf_aes_decrypt.q.out +++ ql/src/test/results/clientpositive/llap/udf_aes_decrypt.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 414243 (type: binary) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_aes_encrypt.q.out ql/src/test/results/clientpositive/llap/udf_aes_encrypt.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_aes_encrypt.q.out rename to ql/src/test/results/clientpositive/llap/udf_aes_encrypt.q.out index 3a96c4bc44..4879bf7009 100644 --- ql/src/test/results/clientpositive/udf_aes_encrypt.q.out +++ ql/src/test/results/clientpositive/llap/udf_aes_encrypt.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CBA4ACFB309839BA426E07D67F23564F (type: binary) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_array.q.out ql/src/test/results/clientpositive/llap/udf_array.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_array.q.out rename to ql/src/test/results/clientpositive/llap/udf_array.q.out index a2ebabaaff..d218f41b7a 100644 --- ql/src/test/results/clientpositive/udf_array.q.out +++ ql/src/test/results/clientpositive/llap/udf_array.q.out @@ -31,11 +31,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: array() (type: array), array()[1] (type: string), array(1,2,3) (type: array), array(1,2,3)[2] (type: int), array(1,'a',2,3) (type: array), array(1,'a',2,3)[2] (type: string), array(array(1),array(2),array(3),array(4))[1][0] (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 500 Data size: 624000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT array(), array()[1], array(1, 2, 3), array(1, 2, 3)[2], array(1,"a", 2, 3), array(1,"a", 2, 3)[2], diff --git ql/src/test/results/clientpositive/udf_ascii.q.out ql/src/test/results/clientpositive/llap/udf_ascii.q.out similarity index 90% rename from ql/src/test/results/clientpositive/udf_ascii.q.out rename to ql/src/test/results/clientpositive/llap/udf_ascii.q.out index 42dd231560..d0dd7a6ce4 100644 --- ql/src/test/results/clientpositive/udf_ascii.q.out +++ ql/src/test/results/clientpositive/llap/udf_ascii.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 70 (type: int), 0 (type: int), 33 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 6000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_between.q.out ql/src/test/results/clientpositive/llap/udf_between.q.out similarity index 82% rename from ql/src/test/results/clientpositive/udf_between.q.out rename to ql/src/test/results/clientpositive/llap/udf_between.q.out index 1a4eb076ef..e8fb4a8350 100644 --- ql/src/test/results/clientpositive/udf_between.q.out +++ ql/src/test/results/clientpositive/llap/udf_between.q.out @@ -29,17 +29,13 @@ STAGE PLANS: TableScan alias: src filterExpr: (UDFToDouble(key) + 100.0D) BETWEEN 100.0D AND 200.0D (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) + 100.0D) BETWEEN 100.0D AND 200.0D (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM src where key + 100 between (150 + -50) AND (150 + 50) LIMIT 20 @@ -89,17 +85,13 @@ STAGE PLANS: TableScan alias: src filterExpr: (UDFToDouble(key) + 100.0D) NOT BETWEEN 100.0D AND 200.0D (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) + 100.0D) NOT BETWEEN 100.0D AND 200.0D (type: boolean) - Statistics: Num rows: 445 Data size: 79210 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 445 Data size: 79210 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM src where key + 100 not between (150 + -50) AND (150 + 50) LIMIT 20 @@ -148,14 +140,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM src where 'b' between 'a' AND 'c' LIMIT 1 @@ -185,14 +174,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM src where 2 between 2 AND '3' LIMIT 1 @@ -257,14 +243,11 @@ STAGE PLANS: TableScan alias: t filterExpr: (i BETWEEN 8 AND 9 or i BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (i BETWEEN 8 AND 9 or i BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM t where i between 8 and 9 @@ -303,14 +286,11 @@ STAGE PLANS: TableScan alias: t filterExpr: (i BETWEEN 6 AND 7 or i BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (i BETWEEN 6 AND 7 or i BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM t where i between 6 and 7 @@ -350,14 +330,11 @@ STAGE PLANS: TableScan alias: t filterExpr: (i NOT BETWEEN 6 AND 7 and i NOT BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (i NOT BETWEEN 6 AND 7 and i NOT BETWEEN 9 AND 10) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT * FROM t where i not between 6 and 7 diff --git ql/src/test/results/clientpositive/udf_bitwise_shiftleft.q.out ql/src/test/results/clientpositive/llap/udf_bitwise_shiftleft.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_bitwise_shiftleft.q.out rename to ql/src/test/results/clientpositive/llap/udf_bitwise_shiftleft.q.out index 0dd68b74b7..c81eb0a851 100644 --- ql/src/test/results/clientpositive/udf_bitwise_shiftleft.q.out +++ ql/src/test/results/clientpositive/llap/udf_bitwise_shiftleft.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 8 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_bitwise_shiftright.q.out ql/src/test/results/clientpositive/llap/udf_bitwise_shiftright.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_bitwise_shiftright.q.out rename to ql/src/test/results/clientpositive/llap/udf_bitwise_shiftright.q.out index 9b3b17d4a9..6a7db86c48 100644 --- ql/src/test/results/clientpositive/udf_bitwise_shiftright.q.out +++ ql/src/test/results/clientpositive/llap/udf_bitwise_shiftright.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_bitwise_shiftrightunsigned.q.out ql/src/test/results/clientpositive/llap/udf_bitwise_shiftrightunsigned.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_bitwise_shiftrightunsigned.q.out rename to ql/src/test/results/clientpositive/llap/udf_bitwise_shiftrightunsigned.q.out index be499580ab..5a150dfef8 100644 --- ql/src/test/results/clientpositive/udf_bitwise_shiftrightunsigned.q.out +++ ql/src/test/results/clientpositive/llap/udf_bitwise_shiftrightunsigned.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_case.q.out ql/src/test/results/clientpositive/llap/udf_case.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_case.q.out rename to ql/src/test/results/clientpositive/llap/udf_case.q.out index 60a348e530..49fc66c56a 100644 --- ql/src/test/results/clientpositive/udf_case.q.out +++ ql/src/test/results/clientpositive/llap/udf_case.q.out @@ -93,11 +93,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int), 5 (type: int), 15 (type: int), null (type: int), 20 (type: int), 24 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 10004 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT CASE 1 diff --git ql/src/test/results/clientpositive/llap/udf_case_column_pruning.q.out ql/src/test/results/clientpositive/llap/udf_case_column_pruning.q.out new file mode 100644 index 0000000000..770c23178b --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf_case_column_pruning.q.out @@ -0,0 +1,163 @@ +PREHOOK: query: EXPLAIN +SELECT CASE a.key + WHEN '1' THEN 2 + WHEN '3' THEN 4 + ELSE 5 + END as key +FROM src a JOIN src b +ON a.key = b.key +ORDER BY key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT CASE a.key + WHEN '1' THEN 2 + WHEN '3' THEN 4 + ELSE 5 + END as key +FROM src a JOIN src b +ON a.key = b.key +ORDER BY key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), CASE WHEN ((key = '1')) THEN (2) WHEN ((key = '3')) THEN (4) ELSE (5) END (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: + + keys: _col1 (type: int) + null sort order: z + Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE + top n: 10 + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: SELECT CASE a.key + WHEN '1' THEN 2 + WHEN '3' THEN 4 + ELSE 5 + END as key +FROM src a JOIN src b +ON a.key = b.key +ORDER BY key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT CASE a.key + WHEN '1' THEN 2 + WHEN '3' THEN 4 + ELSE 5 + END as key +FROM src a JOIN src b +ON a.key = b.key +ORDER BY key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +5 diff --git ql/src/test/results/clientpositive/udf_case_thrift.q.out ql/src/test/results/clientpositive/llap/udf_case_thrift.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_case_thrift.q.out rename to ql/src/test/results/clientpositive/llap/udf_case_thrift.q.out index e540ff9fe9..3a70d8b4d6 100644 --- ql/src/test/results/clientpositive/udf_case_thrift.q.out +++ ql/src/test/results/clientpositive/llap/udf_case_thrift.q.out @@ -49,11 +49,9 @@ STAGE PLANS: TableScan alias: src_thrift Row Limit Per Split: 3 - Statistics: Num rows: 11 Data size: 22440 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: CASE (lint[0]) WHEN (0) THEN ((lint[0] + 1)) WHEN (1) THEN ((lint[0] + 2)) WHEN (2) THEN (100) ELSE (5) END (type: int), CASE (lstring[0]) WHEN ('0') THEN ('zero') WHEN ('10') THEN (concat(lstring[0], ' is ten')) ELSE ('default') END (type: string), CASE (lstring[0]) WHEN ('0') THEN (lstring) ELSE (null) END[0] (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 11 Data size: 22440 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT CASE src_thrift.lint[0] diff --git ql/src/test/results/clientpositive/udf_cbrt.q.out ql/src/test/results/clientpositive/llap/udf_cbrt.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udf_cbrt.q.out rename to ql/src/test/results/clientpositive/llap/udf_cbrt.q.out index fd02051240..45461af386 100644 --- ql/src/test/results/clientpositive/udf_cbrt.q.out +++ ql/src/test/results/clientpositive/llap/udf_cbrt.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 3.0D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_character_length.q.out ql/src/test/results/clientpositive/llap/udf_character_length.q.out similarity index 64% rename from ql/src/test/results/clientpositive/udf_character_length.q.out rename to ql/src/test/results/clientpositive/llap/udf_character_length.q.out index e854005845..e4c22c6567 100644 --- ql/src/test/results/clientpositive/udf_character_length.q.out +++ ql/src/test/results/clientpositive/llap/udf_character_length.q.out @@ -48,70 +48,70 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1_n97 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: character_length(value) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n97 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: len - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(len, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: character_length(value) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n97 + Select Operator + expressions: _col0 (type: int) + outputColumnNames: len + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(len, 'hll') + minReductionHashAggr: 0.96 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -123,7 +123,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n97 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -131,36 +131,6 @@ STAGE PLANS: Column Types: int Table: default.dest1_n97 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n97 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n97 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1_n97 SELECT character_length(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 @@ -254,11 +224,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n97 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: character_length(name) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT character_length(dest1_n97.name) FROM dest1_n97 @@ -289,11 +257,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n97 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: character_length(name) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT char_length(dest1_n97.name) FROM dest1_n97 diff --git ql/src/test/results/clientpositive/udf_concat_insert1.q.out ql/src/test/results/clientpositive/llap/udf_concat_insert1.q.out similarity index 100% rename from ql/src/test/results/clientpositive/udf_concat_insert1.q.out rename to ql/src/test/results/clientpositive/llap/udf_concat_insert1.q.out diff --git ql/src/test/results/clientpositive/udf_concat_ws.q.out ql/src/test/results/clientpositive/llap/udf_concat_ws.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_concat_ws.q.out rename to ql/src/test/results/clientpositive/llap/udf_concat_ws.q.out index 34bb9b30dc..71b5acec9f 100644 --- ql/src/test/results/clientpositive/udf_concat_ws.q.out +++ ql/src/test/results/clientpositive/llap/udf_concat_ws.q.out @@ -58,11 +58,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n8 - Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: concat_ws(c1, c2, c3) (type: string), concat_ws(',', c1, c2, c3) (type: string), concat_ws(null, c1, c2, c3) (type: string), concat_ws('**', c1, null, c3) (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT concat_ws(dest1_n8.c1, dest1_n8.c2, dest1_n8.c3), @@ -113,11 +111,9 @@ STAGE PLANS: TableScan alias: dest1_n8 Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: concat_ws('.', array('www','face','book','com'), '1234') (type: string), concat_ws('-', 'www', array('face','book','com'), '1234') (type: string), concat_ws('F', 'www', array('face','book','com','1234')) (type: string), concat_ws('_', array('www','face'), array('book','com','1234')) (type: string), concat_ws('**', 'www', array('face'), array('book','com','1234')) (type: string), concat_ws('[]', array('www'), 'face', array('book','com','1234')) (type: string), concat_ws('AAA', array('www'), array('face','book','com'), '1234') (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 758 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT concat_ws('.', array('www', 'face', 'book', 'com'), '1234'), diff --git ql/src/test/results/clientpositive/udf_crc32.q.out ql/src/test/results/clientpositive/llap/udf_crc32.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_crc32.q.out rename to ql/src/test/results/clientpositive/llap/udf_crc32.q.out index ef48075255..fa6e4c37ac 100644 --- ql/src/test/results/clientpositive/udf_crc32.q.out +++ ql/src/test/results/clientpositive/llap/udf_crc32.q.out @@ -34,11 +34,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2743272264L (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_current_database.q.out ql/src/test/results/clientpositive/llap/udf_current_database.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udf_current_database.q.out rename to ql/src/test/results/clientpositive/llap/udf_current_database.q.out index 96561e67da..7597a88400 100644 --- ql/src/test/results/clientpositive/udf_current_database.q.out +++ ql/src/test/results/clientpositive/llap/udf_current_database.q.out @@ -24,11 +24,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'default' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select current_database() @@ -73,11 +71,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'xxx' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select current_database() @@ -116,11 +112,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'default' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select current_database() @@ -159,11 +153,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'xxx' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select current_database() diff --git ql/src/test/results/clientpositive/udf_date_format.q.out ql/src/test/results/clientpositive/llap/udf_date_format.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_date_format.q.out rename to ql/src/test/results/clientpositive/llap/udf_date_format.q.out index 928948c628..034bfe35fd 100644 --- ql/src/test/results/clientpositive/udf_date_format.q.out +++ ql/src/test/results/clientpositive/llap/udf_date_format.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'Wednesday' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_datetime_legacy_hybrid_calendar.q.out ql/src/test/results/clientpositive/llap/udf_datetime_legacy_hybrid_calendar.q.out similarity index 83% rename from ql/src/test/results/clientpositive/udf_datetime_legacy_hybrid_calendar.q.out rename to ql/src/test/results/clientpositive/llap/udf_datetime_legacy_hybrid_calendar.q.out index bd22442d98..a505a25630 100644 --- ql/src/test/results/clientpositive/udf_datetime_legacy_hybrid_calendar.q.out +++ ql/src/test/results/clientpositive/llap/udf_datetime_legacy_hybrid_calendar.q.out @@ -109,23 +109,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: datetime_legacy_hybrid_calendar - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: dt (type: date), datetime_legacy_hybrid_calendar(dt) (type: date), ts (type: timestamp), datetime_legacy_hybrid_calendar(ts) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: datetime_legacy_hybrid_calendar + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dt (type: date), datetime_legacy_hybrid_calendar(dt) (type: date), ts (type: timestamp), datetime_legacy_hybrid_calendar(ts) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/udf_decode.q.out ql/src/test/results/clientpositive/llap/udf_decode.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_decode.q.out rename to ql/src/test/results/clientpositive/llap/udf_decode.q.out index bb6cd7804f..6643309fd8 100644 --- ql/src/test/results/clientpositive/udf_decode.q.out +++ ql/src/test/results/clientpositive/llap/udf_decode.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'TestDecode1' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_degrees.q.out ql/src/test/results/clientpositive/llap/udf_degrees.q.out similarity index 88% rename from ql/src/test/results/clientpositive/udf_degrees.q.out rename to ql/src/test/results/clientpositive/llap/udf_degrees.q.out index 1440c5f09e..b556224a44 100644 --- ql/src/test/results/clientpositive/udf_degrees.q.out +++ ql/src/test/results/clientpositive/llap/udf_degrees.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 180.0D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select degrees(PI()) FROM src tablesample (1 rows) @@ -72,11 +70,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 180.0D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select degrees(PI()) FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_elt.q.out ql/src/test/results/clientpositive/llap/udf_elt.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_elt.q.out rename to ql/src/test/results/clientpositive/llap/udf_elt.q.out index 2e7fb9f78d..c17af3d416 100644 --- ql/src/test/results/clientpositive/udf_elt.q.out +++ ql/src/test/results/clientpositive/llap/udf_elt.q.out @@ -56,11 +56,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 500 Data size: 353752 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT elt(2, 'abc', 'defg'), diff --git ql/src/test/results/clientpositive/udf_example_add.q.out ql/src/test/results/clientpositive/llap/udf_example_add.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udf_example_add.q.out rename to ql/src/test/results/clientpositive/llap/udf_example_add.q.out index 43b4eddbff..79c568cbd2 100644 --- ql/src/test/results/clientpositive/udf_example_add.q.out +++ ql/src/test/results/clientpositive/llap/udf_example_add.q.out @@ -38,14 +38,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003D (type: double), 6.6D (type: double), 11.0D (type: double), 10.4D (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT example_add(1, 2), diff --git ql/src/test/results/clientpositive/llap/udf_explode.q.out ql/src/test/results/clientpositive/llap/udf_explode.q.out new file mode 100644 index 0000000000..700efbccf3 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf_explode.q.out @@ -0,0 +1,719 @@ +PREHOOK: query: DESCRIBE FUNCTION explode +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION explode +POSTHOOK: type: DESCFUNCTION +explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns +PREHOOK: query: DESCRIBE FUNCTION EXTENDED explode +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED explode +POSTHOOK: type: DESCFUNCTION +explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns +Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode +Function type:BUILTIN +PREHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Row Limit Per Split: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Reduce Output Operator + bucketingVersion: 2 + key expressions: col (type: int) + null sort order: z + numBuckets: -1 + sort order: + + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: false + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0 + columns.types int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Row Limit Per Split: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Group By Operator + aggregations: count(1) + keys: col (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int) + null sort order: z + numBuckets: -1 + sort order: + + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: false + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types int:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows) ORDER BY myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 +2 +3 +PREHOOK: query: SELECT explode(array(1, 2, 3)) AS (myCol) FROM src tablesample (1 rows) ORDER BY myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(array(1, 2, 3)) AS (myCol) FROM src tablesample (1 rows) ORDER BY myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 +2 +3 +PREHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol ORDER BY a.myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 1 +2 1 +3 1 +PREHOOK: query: EXPLAIN EXTENDED SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Row Limit Per Split: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: map(1:'one',2:'two',3:'three') (type: map) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Reduce Output Operator + bucketingVersion: 2 + key expressions: key (type: int), value (type: string) + null sort order: zz + numBuckets: -1 + sort order: ++ + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: false + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, value) a GROUP BY a.key, a.val ORDER BY a.key, a.val +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, value) a GROUP BY a.key, a.val ORDER BY a.key, a.val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Row Limit Per Split: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: map(1:'one',2:'two',3:'three') (type: map) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Group By Operator + aggregations: count(1) + keys: key (type: int), value (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + numBuckets: -1 + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col2 (type: bigint) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + numBuckets: -1 + sort order: ++ + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col2 (type: bigint) + auto parallelism: false + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2 + columns.types int:string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 one +2 two +3 three +PREHOOK: query: SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val) a GROUP BY a.key, a.val ORDER BY a.key, a.val +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) AS (key, val) FROM src tablesample (1 rows) ORDER BY key, val) a GROUP BY a.key, a.val ORDER BY a.key, a.val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 one 1 +2 two 1 +3 three 1 +PREHOOK: query: drop table lazy_array_map +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table lazy_array_map +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table lazy_array_map (map_col map, array_col array) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_array_map +POSTHOOK: query: create table lazy_array_map (map_col map, array_col array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_array_map +PREHOOK: query: INSERT OVERWRITE TABLE lazy_array_map select map(1, 'one', 2, 'two', 3, 'three'), array('100', '200', '300') FROM src tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@lazy_array_map +POSTHOOK: query: INSERT OVERWRITE TABLE lazy_array_map select map(1, 'one', 2, 'two', 3, 'three'), array('100', '200', '300') FROM src tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@lazy_array_map +POSTHOOK: Lineage: lazy_array_map.array_col EXPRESSION [] +POSTHOOK: Lineage: lazy_array_map.map_col EXPRESSION [] +PREHOOK: query: SELECT array_col, myCol FROM lazy_array_map lateral view explode(array_col) X AS myCol ORDER BY array_col, myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_array_map +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_col, myCol FROM lazy_array_map lateral view explode(array_col) X AS myCol ORDER BY array_col, myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_array_map +#### A masked pattern was here #### +["100","200","300"] 100 +["100","200","300"] 200 +["100","200","300"] 300 +PREHOOK: query: SELECT map_col, myKey, myValue FROM lazy_array_map lateral view explode(map_col) X AS myKey, myValue ORDER BY map_col, myKey, myValue +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_array_map +#### A masked pattern was here #### +POSTHOOK: query: SELECT map_col, myKey, myValue FROM lazy_array_map lateral view explode(map_col) X AS myKey, myValue ORDER BY map_col, myKey, myValue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_array_map +#### A masked pattern was here #### +{1:"one",2:"two",3:"three"} 1 one +{1:"one",2:"two",3:"three"} 2 two +{1:"one",2:"two",3:"three"} 3 three diff --git ql/src/test/results/clientpositive/udf_factorial.q.out ql/src/test/results/clientpositive/llap/udf_factorial.q.out similarity index 90% rename from ql/src/test/results/clientpositive/udf_factorial.q.out rename to ql/src/test/results/clientpositive/llap/udf_factorial.q.out index d00f92c703..74c8cd53df 100644 --- ql/src/test/results/clientpositive/udf_factorial.q.out +++ ql/src/test/results/clientpositive/llap/udf_factorial.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 120L (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_find_in_set.q.out ql/src/test/results/clientpositive/llap/udf_find_in_set.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_find_in_set.q.out rename to ql/src/test/results/clientpositive/llap/udf_find_in_set.q.out index e12c0bd029..f166e89243 100644 --- ql/src/test/results/clientpositive/udf_find_in_set.q.out +++ ql/src/test/results/clientpositive/llap/udf_find_in_set.q.out @@ -36,11 +36,9 @@ STAGE PLANS: Processor Tree: TableScan alias: src1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: find_in_set(key, concat(key, ',', value)) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: FROM src1 SELECT find_in_set(src1.key,concat(src1.key,',',src1.value)) diff --git ql/src/test/results/clientpositive/llap/udf_folder_constants.q.out ql/src/test/results/clientpositive/llap/udf_folder_constants.q.out new file mode 100644 index 0000000000..b7ee1ed049 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf_folder_constants.q.out @@ -0,0 +1,148 @@ +PREHOOK: query: drop table if exists udf_tb1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists udf_tb1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists udf_tb2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists udf_tb2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table udf_tb1 (year int, month int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@udf_tb1 +POSTHOOK: query: create table udf_tb1 (year int, month int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@udf_tb1 +PREHOOK: query: create table udf_tb2(month int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@udf_tb2 +POSTHOOK: query: create table udf_tb2(month int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@udf_tb2 +PREHOOK: query: insert into udf_tb1 values(2001, 11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@udf_tb1 +POSTHOOK: query: insert into udf_tb1 values(2001, 11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@udf_tb1 +POSTHOOK: Lineage: udf_tb1.month SCRIPT [] +POSTHOOK: Lineage: udf_tb1.year SCRIPT [] +PREHOOK: query: insert into udf_tb2 values(11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@udf_tb2 +POSTHOOK: query: insert into udf_tb2 values(11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@udf_tb2 +POSTHOOK: Lineage: udf_tb2.month SCRIPT [] +PREHOOK: query: explain +select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) +PREHOOK: type: QUERY +PREHOOK: Input: default@udf_tb1 +PREHOOK: Input: default@udf_tb2 +#### A masked pattern was here #### +POSTHOOK: query: explain +select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@udf_tb1 +POSTHOOK: Input: default@udf_tb2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: udf_tb1 + filterExpr: ((year = 2001) and month is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((year = 2001) and month is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: month (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: month is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: month is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: month (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 978336000L (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) +PREHOOK: type: QUERY +PREHOOK: Input: default@udf_tb1 +PREHOOK: Input: default@udf_tb2 +#### A masked pattern was here #### +POSTHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@udf_tb1 +POSTHOOK: Input: default@udf_tb2 +#### A masked pattern was here #### +978336000 diff --git ql/src/test/results/clientpositive/udf_format_number.q.out ql/src/test/results/clientpositive/llap/udf_format_number.q.out similarity index 98% rename from ql/src/test/results/clientpositive/udf_format_number.q.out rename to ql/src/test/results/clientpositive/llap/udf_format_number.q.out index 7ca765757a..83a9f902d1 100644 --- ql/src/test/results/clientpositive/udf_format_number.q.out +++ ql/src/test/results/clientpositive/llap/udf_format_number.q.out @@ -50,11 +50,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '12,332.1235' (type: string), '12,332.1000' (type: string), '12,332' (type: string), '12332.2' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 185500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT format_number(12332.123456, 4), diff --git ql/src/test/results/clientpositive/udf_from_utc_timestamp.q.out ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_from_utc_timestamp.q.out rename to ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out index b39677349d..aa6a5e73ae 100644 --- ql/src/test/results/clientpositive/udf_from_utc_timestamp.q.out +++ ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out @@ -29,11 +29,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMP'2012-02-11 02:30:00' (type: timestamp) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_get_json_object.q.out ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out similarity index 98% rename from ql/src/test/results/clientpositive/udf_get_json_object.q.out rename to ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out index fd2d5597ef..d2ae71c96b 100644 --- ql/src/test/results/clientpositive/udf_get_json_object.q.out +++ ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out @@ -62,11 +62,9 @@ STAGE PLANS: Processor Tree: TableScan alias: src_json - Statistics: Num rows: 1 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE Select Operator #### A masked pattern was here #### outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT get_json_object(src_json.json, '$') FROM src_json diff --git ql/src/test/results/clientpositive/udf_greatest.q.out ql/src/test/results/clientpositive/llap/udf_greatest.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_greatest.q.out rename to ql/src/test/results/clientpositive/llap/udf_greatest.q.out index be58652fce..cbd33581d5 100644 --- ql/src/test/results/clientpositive/udf_greatest.q.out +++ ql/src/test/results/clientpositive/llap/udf_greatest.q.out @@ -62,11 +62,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'c' (type: string), 'a' (type: string), 'AaA' (type: string), 'AAA' (type: string), '13' (type: string), '2' (type: string), '03' (type: string), '1' (type: string), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 500 Data size: 343048 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT GREATEST('a', 'b', 'c'), diff --git ql/src/test/results/clientpositive/udf_hash.q.out ql/src/test/results/clientpositive/llap/udf_hash.q.out similarity index 93% rename from ql/src/test/results/clientpositive/udf_hash.q.out rename to ql/src/test/results/clientpositive/llap/udf_hash.q.out index 9b6a5ac188..a793d84a97 100644 --- ql/src/test/results/clientpositive/udf_hash.q.out +++ ql/src/test/results/clientpositive/llap/udf_hash.q.out @@ -41,11 +41,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 2 (type: int), 3 (type: int), -1097262584 (type: int), 1067450368 (type: int), 1076887552 (type: int), 51508 (type: int), 96354 (type: int), 1 (type: int), 0 (type: int), 1026 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT hash(CAST(1 AS TINYINT)), hash(CAST(2 AS SMALLINT)), diff --git ql/src/test/results/clientpositive/udf_hour.q.out ql/src/test/results/clientpositive/llap/udf_hour.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udf_hour.q.out rename to ql/src/test/results/clientpositive/llap/udf_hour.q.out index 3eca55dc39..c12117e78d 100644 --- ql/src/test/results/clientpositive/udf_hour.q.out +++ ql/src/test/results/clientpositive/llap/udf_hour.q.out @@ -41,14 +41,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 13 (type: int), 13 (type: int), 0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT hour('2009-08-07 13:14:15'), hour('13:14:15'), hour('2009-08-07') diff --git ql/src/test/results/clientpositive/udf_if.q.out ql/src/test/results/clientpositive/llap/udf_if.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_if.q.out rename to ql/src/test/results/clientpositive/llap/udf_if.q.out index e6615a4f9d..a9f5a8d0ed 100644 --- ql/src/test/results/clientpositive/udf_if.q.out +++ ql/src/test/results/clientpositive/llap/udf_if.q.out @@ -43,11 +43,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), '1' (type: string), 1 (type: int), 1 (type: int), null (type: int), 2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 50504 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(TRUE, 1, 2) AS COL1, @@ -100,11 +98,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 128S (type: smallint), 1.1 (type: decimal(11,1)), 'ABC' (type: string), '12.3' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 145500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT IF(TRUE, CAST(128 AS SMALLINT), CAST(1 AS TINYINT)) AS COL1, diff --git ql/src/test/results/clientpositive/udf_in_file.q.out ql/src/test/results/clientpositive/llap/udf_in_file.q.out similarity index 69% rename from ql/src/test/results/clientpositive/udf_in_file.q.out rename to ql/src/test/results/clientpositive/llap/udf_in_file.q.out index 9a277d03a2..6af4ea1804 100644 --- ql/src/test/results/clientpositive/udf_in_file.q.out +++ ql/src/test/results/clientpositive/llap/udf_in_file.q.out @@ -52,37 +52,21 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@value_src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: value_src - Statistics: Num rows: 1 Data size: 730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: in_file(str_val, '../../data/files/test2.dat') (type: boolean), in_file(ch_val, '../../data/files/test2.dat') (type: boolean), in_file(vch_val, '../../data/files/test2.dat') (type: boolean), in_file(str_val_neg, '../../data/files/test2.dat') (type: boolean), in_file(ch_val_neg, '../../data/files/test2.dat') (type: boolean), in_file(vch_val_neg, '../../data/files/test2.dat') (type: boolean), in_file('303', '../../data/files/test2.dat') (type: boolean), in_file('304', '../../data/files/test2.dat') (type: boolean), in_file(null, '../../data/files/test2.dat') (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 730 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 730 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: 1 Processor Tree: - ListSink + TableScan + alias: value_src + Select Operator + expressions: in_file(str_val, '../../data/files/test2.dat') (type: boolean), in_file(ch_val, '../../data/files/test2.dat') (type: boolean), in_file(vch_val, '../../data/files/test2.dat') (type: boolean), in_file(str_val_neg, '../../data/files/test2.dat') (type: boolean), in_file(ch_val_neg, '../../data/files/test2.dat') (type: boolean), in_file(vch_val_neg, '../../data/files/test2.dat') (type: boolean), in_file('303', '../../data/files/test2.dat') (type: boolean), in_file('304', '../../data/files/test2.dat') (type: boolean), in_file(null, '../../data/files/test2.dat') (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Limit + Number of rows: 1 + ListSink PREHOOK: query: SELECT in_file(str_val, "../../data/files/test2.dat"), in_file(ch_val, "../../data/files/test2.dat"), diff --git ql/src/test/results/clientpositive/udf_inline.q.out ql/src/test/results/clientpositive/llap/udf_inline.q.out similarity index 84% rename from ql/src/test/results/clientpositive/udf_inline.q.out rename to ql/src/test/results/clientpositive/llap/udf_inline.q.out index 12422a4cc7..d93bc85332 100644 --- ql/src/test/results/clientpositive/udf_inline.q.out +++ ql/src/test/results/clientpositive/llap/udf_inline.q.out @@ -33,21 +33,16 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: array(const struct(1,'dude!'),const struct(2,'Wheres'),const struct(3,'my car?')) (type: array>) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 32000 Basic stats: COMPLETE Column stats: COMPLETE UDTF Operator - Statistics: Num rows: 500 Data size: 32000 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator expressions: col1 (type: int), col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT inline( diff --git ql/src/test/results/clientpositive/udf_instr.q.out ql/src/test/results/clientpositive/llap/udf_instr.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_instr.q.out rename to ql/src/test/results/clientpositive/llap/udf_instr.q.out index 67dd28c025..9c59287082 100644 --- ql/src/test/results/clientpositive/udf_instr.q.out +++ ql/src/test/results/clientpositive/llap/udf_instr.q.out @@ -60,11 +60,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 500 Data size: 22008 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT instr('abcd', 'abc'), diff --git ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out rename to ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out index 7e3642ef2c..2654d71e10 100644 --- ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out +++ ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out @@ -50,14 +50,11 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: true (type: boolean), true (type: boolean), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 6000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT NULL IS NULL, @@ -108,17 +105,13 @@ STAGE PLANS: TableScan alias: src_thrift filterExpr: (lint is not null and mstringstring is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (lint is not null and mstringstring is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: lint is not null (type: boolean), lintstring is not null (type: boolean), mstringstring is not null (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 3600 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: FROM src_thrift diff --git ql/src/test/results/clientpositive/llap/udf_isops_simplify.q.out ql/src/test/results/clientpositive/llap/udf_isops_simplify.q.out new file mode 100644 index 0000000000..e7ccd7bc08 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udf_isops_simplify.q.out @@ -0,0 +1,422 @@ +PREHOOK: query: create table t (a integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (a integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: explain select not ((a>0) is not true) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is not true) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (_col0 > 0) is true (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select not ((a>0) is not false) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is not false) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (_col0 > 0) is false (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select not ((a>0) is not null) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is not null) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 is null (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select not ((a>0) is true) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is true) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (_col0 > 0) is not true (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select not ((a>0) is false) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is false) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (_col0 > 0) is not false (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select not ((a>0) is null) from t group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain select not ((a>0) is null) from t group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: a + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 is not null (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/udf_java_method.q.out ql/src/test/results/clientpositive/llap/udf_java_method.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_java_method.q.out rename to ql/src/test/results/clientpositive/llap/udf_java_method.q.out index 71de4a1d36..1389c86fa9 100644 --- ql/src/test/results/clientpositive/udf_java_method.q.out +++ ql/src/test/results/clientpositive/llap/udf_java_method.q.out @@ -48,12 +48,10 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator expressions: reflect('java.lang.String','valueOf',1) (type: string), reflect('java.lang.String','isEmpty') (type: string), reflect('java.lang.Math','max',2,3) (type: string), reflect('java.lang.Math','min',2,3) (type: string), reflect('java.lang.Math','round',2.5D) (type: string), round(reflect('java.lang.Math','exp',1.0D), 6) (type: double), reflect('java.lang.Math','floor',1.9D) (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 500 Data size: 556000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT java_method("java.lang.String", "valueOf", 1), diff --git ql/src/test/results/clientpositive/udf_last_day.q.out ql/src/test/results/clientpositive/llap/udf_last_day.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_last_day.q.out rename to ql/src/test/results/clientpositive/llap/udf_last_day.q.out index 716e39bedc..f9575973fd 100644 --- ql/src/test/results/clientpositive/udf_last_day.q.out +++ ql/src/test/results/clientpositive/llap/udf_last_day.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2015-02-28' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_least.q.out ql/src/test/results/clientpositive/llap/udf_least.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_least.q.out rename to ql/src/test/results/clientpositive/llap/udf_least.q.out index bc867dc88d..11a74878f7 100644 --- ql/src/test/results/clientpositive/udf_least.q.out +++ ql/src/test/results/clientpositive/llap/udf_least.q.out @@ -62,11 +62,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'a' (type: string), 'B' (type: string), 'AAA' (type: string), 'A' (type: string), '11' (type: string), '11' (type: string), '01' (type: string), '01' (type: string), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 500 Data size: 343048 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT LEAST('a', 'b', 'c'), diff --git ql/src/test/results/clientpositive/udf_length.q.out ql/src/test/results/clientpositive/llap/udf_length.q.out similarity index 55% rename from ql/src/test/results/clientpositive/udf_length.q.out rename to ql/src/test/results/clientpositive/llap/udf_length.q.out index f019db9849..7a5bfba7a9 100644 --- ql/src/test/results/clientpositive/udf_length.q.out +++ ql/src/test/results/clientpositive/llap/udf_length.q.out @@ -31,70 +31,70 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1_n134 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: length(value) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n134 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: len - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(len, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: length(value) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n134 + Select Operator + expressions: _col0 (type: int) + outputColumnNames: len + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(len, 'hll') + minReductionHashAggr: 0.96 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,7 +106,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n134 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -114,36 +114,6 @@ STAGE PLANS: Column Types: int Table: default.dest1_n134 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n134 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n134 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1_n134 SELECT length(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 @@ -228,11 +198,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n134 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: length(name) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT length(dest1_n134.name) FROM dest1_n134 diff --git ql/src/test/results/clientpositive/udf_levenshtein.q.out ql/src/test/results/clientpositive/llap/udf_levenshtein.q.out similarity index 93% rename from ql/src/test/results/clientpositive/udf_levenshtein.q.out rename to ql/src/test/results/clientpositive/llap/udf_levenshtein.q.out index 7b8047ba10..a621748d50 100644 --- ql/src/test/results/clientpositive/udf_levenshtein.q.out +++ ql/src/test/results/clientpositive/llap/udf_levenshtein.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_like.q.out ql/src/test/results/clientpositive/llap/udf_like.q.out similarity index 94% rename from ql/src/test/results/clientpositive/udf_like.q.out rename to ql/src/test/results/clientpositive/llap/udf_like.q.out index 8a0bb986eb..3c86fa7fee 100644 --- ql/src/test/results/clientpositive/udf_like.q.out +++ ql/src/test/results/clientpositive/llap/udf_like.q.out @@ -40,14 +40,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: true (type: boolean), false (type: boolean), true (type: boolean), true (type: boolean), false (type: boolean), false (type: boolean), false (type: boolean), false (type: boolean), true (type: boolean), false (type: boolean), false (type: boolean), false (type: boolean), false (type: boolean), true (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 250 Data size: 14000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT '_%_' LIKE '%\_\%\_%', '__' LIKE '%\_\%\_%', '%%_%_' LIKE '%\_\%\_%', '%_%_%' LIKE '%\%\_\%', diff --git ql/src/test/results/clientpositive/udf_locate.q.out ql/src/test/results/clientpositive/llap/udf_locate.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_locate.q.out rename to ql/src/test/results/clientpositive/llap/udf_locate.q.out index cb3f63bd1b..f361c168d0 100644 --- ql/src/test/results/clientpositive/udf_locate.q.out +++ ql/src/test/results/clientpositive/llap/udf_locate.q.out @@ -69,11 +69,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 4 (type: int), 4 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int), 0 (type: int), 0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 500 Data size: 30008 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT locate('abc', 'abcd'), diff --git ql/src/test/results/clientpositive/udf_lower.q.out ql/src/test/results/clientpositive/llap/udf_lower.q.out similarity index 52% rename from ql/src/test/results/clientpositive/udf_lower.q.out rename to ql/src/test/results/clientpositive/llap/udf_lower.q.out index 3e3a1e27e0..53e3de6997 100644 --- ql/src/test/results/clientpositive/udf_lower.q.out +++ ql/src/test/results/clientpositive/llap/udf_lower.q.out @@ -25,38 +25,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'abc 123' (type: string), 'ABC 123' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Select Operator + expressions: 'abc 123' (type: string), 'ABC 123' (type: string) + outputColumnNames: _col0, _col1 + ListSink PREHOOK: query: SELECT lower('AbC 123'), upper('AbC 123') FROM src WHERE key = 86 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/udf_lpad.q.out ql/src/test/results/clientpositive/llap/udf_lpad.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_lpad.q.out rename to ql/src/test/results/clientpositive/llap/udf_lpad.q.out index f7a316e277..5ebfea264b 100644 --- ql/src/test/results/clientpositive/udf_lpad.q.out +++ ql/src/test/results/clientpositive/llap/udf_lpad.q.out @@ -46,11 +46,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'h' (type: string), '...hi' (type: string), '1231hi' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_map.q.out ql/src/test/results/clientpositive/llap/udf_map.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_map.q.out rename to ql/src/test/results/clientpositive/llap/udf_map.q.out index 702feb2d9a..eb47453381 100644 --- ql/src/test/results/clientpositive/udf_map.q.out +++ ql/src/test/results/clientpositive/llap/udf_map.q.out @@ -31,11 +31,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: map() (type: map), map(1:'a',2:'b',3:'c') (type: map), map(1:2,'a':'b') (type: map), map(1:'a',2:'b',3:'c')[2] (type: string), map(1:2,'a':'b')['a'] (type: string), map(1:array('a'))[1][0] (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 857500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT map(), map(1, "a", 2, "b", 3, "c"), map(1, 2, "a", "b"), diff --git ql/src/test/results/clientpositive/udf_mask.q.out ql/src/test/results/clientpositive/llap/udf_mask.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_mask.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask.q.out index f3a5f260cf..31140d3bd7 100644 --- ql/src/test/results/clientpositive/udf_mask.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask.q.out @@ -45,11 +45,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'XxxxXxxxxx1000' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask('TestString-123', 'X', 'x', '0', ':'), diff --git ql/src/test/results/clientpositive/udf_mask_first_n.q.out ql/src/test/results/clientpositive/llap/udf_mask_first_n.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_mask_first_n.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask_first_n.q.out index 107461b18a..9aa9743a42 100644 --- ql/src/test/results/clientpositive/udf_mask_first_n.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask_first_n.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'XxxxString-123' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask_first_n('TestString-123', 4, 'X', 'x', '0', ':'), diff --git ql/src/test/results/clientpositive/udf_mask_hash.q.out ql/src/test/results/clientpositive/llap/udf_mask_hash.q.out similarity index 93% rename from ql/src/test/results/clientpositive/udf_mask_hash.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask_hash.q.out index 6d79a565bb..e489f2efa7 100644 --- ql/src/test/results/clientpositive/udf_mask_hash.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask_hash.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '8b44d559dc5d60e4453c9b4edf2a455fbce054bb8504cd3eb9b5f391bd239c90' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 148 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask_hash('TestString-123'), diff --git ql/src/test/results/clientpositive/udf_mask_last_n.q.out ql/src/test/results/clientpositive/llap/udf_mask_last_n.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_mask_last_n.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask_last_n.q.out index 2119084add..34dcd5b581 100644 --- ql/src/test/results/clientpositive/udf_mask_last_n.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask_last_n.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'TestString1000' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask_last_n('TestString-123', 4, 'X', 'x', '0', ':'), diff --git ql/src/test/results/clientpositive/udf_mask_show_first_n.q.out ql/src/test/results/clientpositive/llap/udf_mask_show_first_n.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_mask_show_first_n.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask_show_first_n.q.out index 918c5b8c7f..332dacbdff 100644 --- ql/src/test/results/clientpositive/udf_mask_show_first_n.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask_show_first_n.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'TestXxxxxx1000' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask_show_first_n('TestString-123', 4, 'X', 'x', '0', ':'), diff --git ql/src/test/results/clientpositive/udf_mask_show_last_n.q.out ql/src/test/results/clientpositive/llap/udf_mask_show_last_n.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_mask_show_last_n.q.out rename to ql/src/test/results/clientpositive/llap/udf_mask_show_last_n.q.out index af06ba7218..4c812a2ebd 100644 --- ql/src/test/results/clientpositive/udf_mask_show_last_n.q.out +++ ql/src/test/results/clientpositive/llap/udf_mask_show_last_n.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'XxxxXxxxxx-123' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select mask_show_last_n('TestString-123', 4, 'X', 'x', '0', ':'), diff --git ql/src/test/results/clientpositive/udf_md5.q.out ql/src/test/results/clientpositive/llap/udf_md5.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_md5.q.out rename to ql/src/test/results/clientpositive/llap/udf_md5.q.out index 16a949eb37..4d3147bbb9 100644 --- ql/src/test/results/clientpositive/udf_md5.q.out +++ ql/src/test/results/clientpositive/llap/udf_md5.q.out @@ -35,11 +35,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '902fbdd2b1df0c4f70b4a5d23525e932' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_minute.q.out ql/src/test/results/clientpositive/llap/udf_minute.q.out similarity index 57% rename from ql/src/test/results/clientpositive/udf_minute.q.out rename to ql/src/test/results/clientpositive/llap/udf_minute.q.out index efb8dc6f7e..d0c74b8f72 100644 --- ql/src/test/results/clientpositive/udf_minute.q.out +++ ql/src/test/results/clientpositive/llap/udf_minute.q.out @@ -31,38 +31,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 14 (type: int), 14 (type: int), 0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Select Operator + expressions: 14 (type: int), 14 (type: int), 0 (type: int) + outputColumnNames: _col0, _col1, _col2 + ListSink PREHOOK: query: SELECT minute('2009-08-07 13:14:15'), minute('13:14:15'), minute('2009-08-07') FROM src WHERE key = 86 diff --git ql/src/test/results/clientpositive/udf_months_between.q.out ql/src/test/results/clientpositive/llap/udf_months_between.q.out similarity index 98% rename from ql/src/test/results/clientpositive/udf_months_between.q.out rename to ql/src/test/results/clientpositive/llap/udf_months_between.q.out index 44a73e8466..9d967d2d18 100644 --- ql/src/test/results/clientpositive/udf_months_between.q.out +++ ql/src/test/results/clientpositive/llap/udf_months_between.q.out @@ -34,11 +34,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1.03225806D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_named_struct.q.out ql/src/test/results/clientpositive/llap/udf_named_struct.q.out similarity index 90% rename from ql/src/test/results/clientpositive/udf_named_struct.q.out rename to ql/src/test/results/clientpositive/llap/udf_named_struct.q.out index 38fd7233b9..f8315b51dc 100644 --- ql/src/test/results/clientpositive/udf_named_struct.q.out +++ ql/src/test/results/clientpositive/llap/udf_named_struct.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: named_struct('foo',1,'bar',2) (type: struct), named_struct('foo',1,'bar',2).foo (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 30000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT named_struct("foo", 1, "bar", 2), diff --git ql/src/test/results/clientpositive/udf_next_day.q.out ql/src/test/results/clientpositive/llap/udf_next_day.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_next_day.q.out rename to ql/src/test/results/clientpositive/llap/udf_next_day.q.out index c4214d7e54..1e37311c33 100644 --- ql/src/test/results/clientpositive/udf_next_day.q.out +++ ql/src/test/results/clientpositive/llap/udf_next_day.q.out @@ -34,11 +34,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-20' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT next_day('2015-01-11', 'su'), diff --git ql/src/test/results/clientpositive/udf_notequal.q.out ql/src/test/results/clientpositive/llap/udf_notequal.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_notequal.q.out rename to ql/src/test/results/clientpositive/llap/udf_notequal.q.out index a959716303..6ac98a53a1 100644 --- ql/src/test/results/clientpositive/udf_notequal.q.out +++ ql/src/test/results/clientpositive/llap/udf_notequal.q.out @@ -49,14 +49,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (key <> '302') (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key <> '302') (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT key, value @@ -595,14 +592,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (key <> '302') (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key <> '302') (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT key, value diff --git ql/src/test/results/clientpositive/udf_nullif.q.out ql/src/test/results/clientpositive/llap/udf_nullif.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_nullif.q.out rename to ql/src/test/results/clientpositive/llap/udf_nullif.q.out index 3251b22f5e..492b412a65 100644 --- ql/src/test/results/clientpositive/udf_nullif.q.out +++ ql/src/test/results/clientpositive/llap/udf_nullif.q.out @@ -31,11 +31,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select nullif(1.0,2.0) @@ -57,11 +55,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: decimal(1,0)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select nullif('y','x') @@ -83,11 +79,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'y' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select nullif(1,1) diff --git ql/src/test/results/clientpositive/udf_nvl.q.out ql/src/test/results/clientpositive/llap/udf_nvl.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udf_nvl.q.out rename to ql/src/test/results/clientpositive/llap/udf_nvl.q.out index d072b9b2e9..9fa74ed367 100644 --- ql/src/test/results/clientpositive/udf_nvl.q.out +++ ql/src/test/results/clientpositive/llap/udf_nvl.q.out @@ -39,11 +39,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT NVL( 1 , 2 ) AS COL1, diff --git ql/src/test/results/clientpositive/udf_octet_length.q.out ql/src/test/results/clientpositive/llap/udf_octet_length.q.out similarity index 59% rename from ql/src/test/results/clientpositive/udf_octet_length.q.out rename to ql/src/test/results/clientpositive/llap/udf_octet_length.q.out index 3e3f523952..f0117794f6 100644 --- ql/src/test/results/clientpositive/udf_octet_length.q.out +++ ql/src/test/results/clientpositive/llap/udf_octet_length.q.out @@ -31,70 +31,70 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1_n165 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: octet_length(value) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n165 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: len - Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(len, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: octet_length(value) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n165 + Select Operator + expressions: _col0 (type: int) + outputColumnNames: len + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(len, 'hll') + minReductionHashAggr: 0.96 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,7 +106,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n165 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -114,36 +114,6 @@ STAGE PLANS: Column Types: int Table: default.dest1_n165 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n165 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n165 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1_n165 SELECT octet_length(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 @@ -237,11 +207,9 @@ STAGE PLANS: Processor Tree: TableScan alias: dest1_n165 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: octet_length(name) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT octet_length(dest1_n165.name) FROM dest1_n165 diff --git ql/src/test/results/clientpositive/udf_parse_url.q.out ql/src/test/results/clientpositive/llap/udf_parse_url.q.out similarity index 75% rename from ql/src/test/results/clientpositive/udf_parse_url.q.out rename to ql/src/test/results/clientpositive/llap/udf_parse_url.q.out index a539da8f22..9e0c666538 100644 --- ql/src/test/results/clientpositive/udf_parse_url.q.out +++ ql/src/test/results/clientpositive/llap/udf_parse_url.q.out @@ -52,38 +52,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'facebook.com' (type: string), '/path1/p.php' (type: string), 'k1=v1&k2=v2' (type: string), 'Ref1' (type: string), 'v2' (type: string), 'v1' (type: string), null (type: string), '/path1/p.php?k1=v1&k2=v2' (type: string), 'http' (type: string), null (type: string), 'facebook.com' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 250 Data size: 209918 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 209918 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 86.0D) (type: boolean) + Select Operator + expressions: 'facebook.com' (type: string), '/path1/p.php' (type: string), 'k1=v1&k2=v2' (type: string), 'Ref1' (type: string), 'v2' (type: string), 'v1' (type: string), null (type: string), '/path1/p.php?k1=v1&k2=v2' (type: string), 'http' (type: string), null (type: string), 'facebook.com' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + ListSink PREHOOK: query: SELECT parse_url('http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1', 'HOST'), parse_url('http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1', 'PATH'), diff --git ql/src/test/results/clientpositive/udf_position.q.out ql/src/test/results/clientpositive/llap/udf_position.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_position.q.out rename to ql/src/test/results/clientpositive/llap/udf_position.q.out index ac5f6c088b..39fb5b30ca 100644 --- ql/src/test/results/clientpositive/udf_position.q.out +++ ql/src/test/results/clientpositive/llap/udf_position.q.out @@ -69,11 +69,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 4 (type: int), 4 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int), 0 (type: int), 0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 500 Data size: 30008 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT position('abc', 'abcd'), diff --git ql/src/test/results/clientpositive/udf_quarter.q.out ql/src/test/results/clientpositive/llap/udf_quarter.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_quarter.q.out rename to ql/src/test/results/clientpositive/llap/udf_quarter.q.out index 8a479946fc..213ebcef5a 100644 --- ql/src/test/results/clientpositive/udf_quarter.q.out +++ ql/src/test/results/clientpositive/llap/udf_quarter.q.out @@ -31,11 +31,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_radians.q.out ql/src/test/results/clientpositive/llap/udf_radians.q.out similarity index 90% rename from ql/src/test/results/clientpositive/udf_radians.q.out rename to ql/src/test/results/clientpositive/llap/udf_radians.q.out index cac8ef98f6..5f9c152722 100644 --- ql/src/test/results/clientpositive/udf_radians.q.out +++ ql/src/test/results/clientpositive/llap/udf_radians.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1.000000357564167D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select radians(57.2958) FROM src tablesample (1 rows) @@ -81,11 +79,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1.000000357564167D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select radians(57.2958) FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_reflect.q.out ql/src/test/results/clientpositive/llap/udf_reflect.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_reflect.q.out rename to ql/src/test/results/clientpositive/llap/udf_reflect.q.out index 57b3b30e6d..eb39d35a33 100644 --- ql/src/test/results/clientpositive/udf_reflect.q.out +++ ql/src/test/results/clientpositive/llap/udf_reflect.q.out @@ -52,12 +52,10 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator expressions: reflect('java.lang.String','valueOf',1) (type: string), reflect('java.lang.String','isEmpty') (type: string), reflect('java.lang.Math','max',2,3) (type: string), reflect('java.lang.Math','min',2,3) (type: string), reflect('java.lang.Math','round',2.5D) (type: string), round(reflect('java.lang.Math','exp',1.0D), 6) (type: double), reflect('java.lang.Math','floor',1.9D) (type: string), reflect('java.lang.Integer','valueOf',key,16) (type: string), reflect('java.lang.Integer','valueOf','16') (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 500 Data size: 740000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT reflect("java.lang.String", "valueOf", 1), diff --git ql/src/test/results/clientpositive/udf_reflect2.q.out ql/src/test/results/clientpositive/llap/udf_reflect2.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_reflect2.q.out rename to ql/src/test/results/clientpositive/llap/udf_reflect2.q.out index 7f03ec74e2..ed82e588b9 100644 --- ql/src/test/results/clientpositive/udf_reflect2.q.out +++ ql/src/test/results/clientpositive/llap/udf_reflect2.q.out @@ -97,15 +97,12 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator expressions: UDFToInteger(key) (type: int), reflect2(UDFToInteger(key),'byteValue') (type: tinyint), reflect2(UDFToInteger(key),'shortValue') (type: smallint), reflect2(UDFToInteger(key),'intValue') (type: int), reflect2(UDFToInteger(key),'longValue') (type: bigint), reflect2(UDFToInteger(key),'floatValue') (type: float), reflect2(UDFToInteger(key),'doubleValue') (type: double), reflect2(UDFToInteger(key),'toString') (type: string), value (type: string), reflect2(value,'concat','_concat') (type: string), reflect2(value,'contains','86') (type: boolean), reflect2(value,'startsWith','v') (type: boolean), reflect2(value,'endsWith','6') (type: boolean), reflect2(value,'equals','val_86') (type: boolean), reflect2(value,'equalsIgnoreCase','VAL_86') (type: boolean), reflect2(value,'getBytes') (type: binary), reflect2(value,'indexOf','1') (type: int), reflect2(value,'lastIndexOf','1') (type: int), reflect2(value,'replace','val','VALUE') (type: string), reflect2(value,'substring',1) (type: string), reflect2(value,'substring',1,5) (type: string), reflect2(value,'toUpperCase') (type: string), reflect2(value,'trim') (type: string), TIMESTAMP'2013-02-15 19:41:20' (type: timestamp), 2013 (type: int), 2 (type: int), 15 (type: int), 19 (type: int), 41 (type: int), 20 (type: int), 1360957280000L (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 500 Data size: 829500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 5 - Statistics: Num rows: 5 Data size: 8295 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT key, diff --git ql/src/test/results/clientpositive/udf_repeat.q.out ql/src/test/results/clientpositive/llap/udf_repeat.q.out similarity index 90% rename from ql/src/test/results/clientpositive/udf_repeat.q.out rename to ql/src/test/results/clientpositive/llap/udf_repeat.q.out index d2a7e51242..9490b7aa33 100644 --- ql/src/test/results/clientpositive/udf_repeat.q.out +++ ql/src/test/results/clientpositive/llap/udf_repeat.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'FacebookFacebookFacebook' (type: string), '' (type: string), '' (type: string), '' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 180000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_reverse.q.out ql/src/test/results/clientpositive/llap/udf_reverse.q.out similarity index 54% rename from ql/src/test/results/clientpositive/udf_reverse.q.out rename to ql/src/test/results/clientpositive/llap/udf_reverse.q.out index d2ecbd0153..03e3a2126c 100644 --- ql/src/test/results/clientpositive/udf_reverse.q.out +++ ql/src/test/results/clientpositive/llap/udf_reverse.q.out @@ -31,70 +31,70 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1_n44 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: reverse(value) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n44 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: len - Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(len, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: reverse(value) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n44 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: len + Statistics: Num rows: 25 Data size: 4600 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(len, 'hll') + minReductionHashAggr: 0.96 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,7 +106,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n44 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -114,36 +114,6 @@ STAGE PLANS: Column Types: string Table: default.dest1_n44 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n44 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n44 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1_n44 SELECT reverse(src1.value) PREHOOK: type: QUERY PREHOOK: Input: default@src1 diff --git ql/src/test/results/clientpositive/udf_rpad.q.out ql/src/test/results/clientpositive/llap/udf_rpad.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_rpad.q.out rename to ql/src/test/results/clientpositive/llap/udf_rpad.q.out index 80bd5d0778..82ab2f6197 100644 --- ql/src/test/results/clientpositive/udf_rpad.q.out +++ ql/src/test/results/clientpositive/llap/udf_rpad.q.out @@ -46,11 +46,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'h' (type: string), 'hi...' (type: string), 'hi1231' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 132000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_second.q.out ql/src/test/results/clientpositive/llap/udf_second.q.out similarity index 87% rename from ql/src/test/results/clientpositive/udf_second.q.out rename to ql/src/test/results/clientpositive/llap/udf_second.q.out index b60b7af4f6..19a6798129 100644 --- ql/src/test/results/clientpositive/udf_second.q.out +++ ql/src/test/results/clientpositive/llap/udf_second.q.out @@ -41,14 +41,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 15 (type: int), 15 (type: int), 0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT second('2009-08-07 13:14:15'), second('13:14:15'), second('2009-08-07') diff --git ql/src/test/results/clientpositive/udf_sha1.q.out ql/src/test/results/clientpositive/llap/udf_sha1.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_sha1.q.out rename to ql/src/test/results/clientpositive/llap/udf_sha1.q.out index 1fc83242d8..9add9eda41 100644 --- ql/src/test/results/clientpositive/udf_sha1.q.out +++ ql/src/test/results/clientpositive/llap/udf_sha1.q.out @@ -35,11 +35,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_sha2.q.out ql/src/test/results/clientpositive/llap/udf_sha2.q.out similarity index 96% rename from ql/src/test/results/clientpositive/udf_sha2.q.out rename to ql/src/test/results/clientpositive/llap/udf_sha2.q.out index f8292c213a..c90d5db579 100644 --- ql/src/test/results/clientpositive/udf_sha2.q.out +++ ql/src/test/results/clientpositive/llap/udf_sha2.q.out @@ -32,11 +32,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 148 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_sign.q.out ql/src/test/results/clientpositive/llap/udf_sign.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_sign.q.out rename to ql/src/test/results/clientpositive/llap/udf_sign.q.out index 91ef2a6606..caa5f7fcb2 100644 --- ql/src/test/results/clientpositive/udf_sign.q.out +++ ql/src/test/results/clientpositive/llap/udf_sign.q.out @@ -19,11 +19,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 0.0D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select sign(0) FROM src tablesample (1 rows) @@ -89,11 +87,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 0.0D (type: double) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select sign(0) FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_size.q.out ql/src/test/results/clientpositive/llap/udf_size.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udf_size.q.out rename to ql/src/test/results/clientpositive/llap/udf_size.q.out index 938e170541..a28d23e992 100644 --- ql/src/test/results/clientpositive/udf_size.q.out +++ ql/src/test/results/clientpositive/llap/udf_size.q.out @@ -43,17 +43,13 @@ STAGE PLANS: TableScan alias: src_thrift filterExpr: (lint is not null and mstringstring is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (lint is not null and mstringstring is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int), -1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 39600 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 3600 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: FROM src_thrift diff --git ql/src/test/results/clientpositive/udf_sort_array.q.out ql/src/test/results/clientpositive/llap/udf_sort_array.q.out similarity index 97% rename from ql/src/test/results/clientpositive/udf_sort_array.q.out rename to ql/src/test/results/clientpositive/llap/udf_sort_array.q.out index d0e5362e2e..972399d2a7 100644 --- ql/src/test/results/clientpositive/udf_sort_array.q.out +++ ql/src/test/results/clientpositive/llap/udf_sort_array.q.out @@ -40,11 +40,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: sort_array(array('b','d','c','a')) (type: array) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 388000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT sort_array(array("f", "a", "g", "c", "b", "d", "e")) FROM src tablesample (1 rows) diff --git ql/src/test/results/clientpositive/udf_sort_array_by.q.out ql/src/test/results/clientpositive/llap/udf_sort_array_by.q.out similarity index 100% rename from ql/src/test/results/clientpositive/udf_sort_array_by.q.out rename to ql/src/test/results/clientpositive/llap/udf_sort_array_by.q.out index edd61b8ad6..4eda558686 100644 --- ql/src/test/results/clientpositive/udf_sort_array_by.q.out +++ ql/src/test/results/clientpositive/llap/udf_sort_array_by.q.out @@ -112,8 +112,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'salary') as sing POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] Facebook US [{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] +Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] PREHOOK: query: select company,country,sort_array_by(employee,'salary','ASC') as single_field_sort from sort_array_by_table PREHOOK: type: QUERY @@ -123,8 +123,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'salary','ASC') a POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] Facebook US [{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] +Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] PREHOOK: query: select company,country,sort_array_by(employee,'salary','desc') as single_field_sort from sort_array_by_table PREHOOK: type: QUERY @@ -134,8 +134,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'salary','desc') POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000}] Facebook US [{"empid":206,"name":"Keiko","age":41,"salary":80500},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":310,"name":"Ben","age":31,"salary":21000}] +Google IN [{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000}] Microsoft UK [{"empid":300,"name":"Spiro","age":38,"salary":80300},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":730,"name":"Eden","age":45,"salary":20300}] PREHOOK: query: select company,country, sort_array_by(employee,'salary') as single_field_sort, @@ -153,8 +153,8 @@ from sort_array_by_table POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] [{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000}] Facebook US [{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] [{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] [{"empid":206,"name":"Keiko","age":41,"salary":80500},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":310,"name":"Ben","age":31,"salary":21000}] +Google IN [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] [{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":800,"name":"Able","age":28,"salary":80000}] [{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] [{"empid":300,"name":"Spiro","age":38,"salary":80300},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":730,"name":"Eden","age":45,"salary":20300}] PREHOOK: query: select company,country,sort_array_by(employee,'name','salary') as multiple_field_sort from sort_array_by_table PREHOOK: type: QUERY @@ -164,8 +164,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'name','salary') POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] Facebook US [{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] +Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] PREHOOK: query: select company,country,sort_array_by(employee,'name','salary','asc') as multiple_field_sort from sort_array_by_table PREHOOK: type: QUERY @@ -175,8 +175,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'name','salary',' POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] Facebook US [{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] +Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] PREHOOK: query: select company,country,sort_array_by(employee,'name',"salary","DESC") as multiple_field_sort from sort_array_by_table PREHOOK: type: QUERY @@ -186,8 +186,8 @@ POSTHOOK: query: select company,country,sort_array_by(employee,'name',"salary"," POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":756,"name":"Able","age":23,"salary":76889}] Facebook US [{"empid":206,"name":"Keiko","age":41,"salary":80500},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":700,"name":"Aron","age":21,"salary":50000}] +Google IN [{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":756,"name":"Able","age":23,"salary":76889}] Microsoft UK [{"empid":300,"name":"Spiro","age":38,"salary":80300},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":730,"name":"Eden","age":45,"salary":20300}] PREHOOK: query: select company,country, sort_array_by(employee,'name','salary') as multiple_field_sort, @@ -205,8 +205,8 @@ from sort_array_by_table POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table #### A masked pattern was here #### -Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] [{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":756,"name":"Able","age":23,"salary":76889}] Facebook US [{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] [{"empid":700,"name":"Aron","age":21,"salary":50000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":206,"name":"Keiko","age":41,"salary":80500}] [{"empid":206,"name":"Keiko","age":41,"salary":80500},{"empid":200,"name":"Keiko","age":28,"salary":80000},{"empid":390,"name":"Ben","age":21,"salary":70000},{"empid":310,"name":"Ben","age":31,"salary":21000},{"empid":320,"name":"Aron","age":18,"salary":70000},{"empid":700,"name":"Aron","age":21,"salary":50000}] +Google IN [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] [{"empid":756,"name":"Able","age":23,"salary":76889},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":900,"name":"Hary","age":21,"salary":50000}] [{"empid":900,"name":"Hary","age":21,"salary":50000},{"empid":76,"name":"Hary","age":87,"salary":10000},{"empid":130,"name":"Boo","age":22,"salary":79000},{"empid":100,"name":"Boo","age":21,"salary":70000},{"empid":800,"name":"Able","age":28,"salary":80000},{"empid":756,"name":"Able","age":23,"salary":76889}] Microsoft UK [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] [{"empid":730,"name":"Eden","age":45,"salary":20300},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":300,"name":"Spiro","age":38,"salary":80300}] [{"empid":300,"name":"Spiro","age":38,"salary":80300},{"empid":900,"name":"Spiro","age":28,"salary":80000},{"empid":600,"name":"James","age":21,"salary":70000},{"empid":313,"name":"James","age":11,"salary":30000},{"empid":260,"name":"Eden","age":31,"salary":50020},{"empid":730,"name":"Eden","age":45,"salary":20300}] PREHOOK: query: DROP TABLE IF EXISTS sort_array_by_order_name PREHOOK: type: DROPTABLE @@ -319,12 +319,12 @@ POSTHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW exp POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table_order #### A masked pattern was here #### -{"name":"Hary","age":87} -{"name":"Hary","age":21} -{"name":"Boo","age":22} -{"name":"Boo","age":21} {"name":"Able","age":23} {"name":"Able","age":28} +{"name":"Boo","age":21} +{"name":"Boo","age":22} +{"name":"Hary","age":21} +{"name":"Hary","age":87} PREHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW explode(sort_array_by(a_struct_array, 'name', 'DESC')) structTable AS a_struct PREHOOK: type: QUERY PREHOOK: Input: default@sort_array_by_table_order @@ -333,12 +333,12 @@ POSTHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW exp POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table_order #### A masked pattern was here #### -{"name":"Hary","age":21} -{"name":"Hary","age":87} +{"name":"Able","age":23} +{"name":"Able","age":28} {"name":"Boo","age":21} {"name":"Boo","age":22} -{"name":"Able","age":28} -{"name":"Able","age":23} +{"name":"Hary","age":21} +{"name":"Hary","age":87} PREHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW explode(a_struct_array) structTable AS a_struct ORDER BY a_struct.name DESC,a_struct.age DESC PREHOOK: type: QUERY PREHOOK: Input: default@sort_array_by_table_order @@ -347,12 +347,12 @@ POSTHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW exp POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table_order #### A masked pattern was here #### -{"name":"Hary","age":87} -{"name":"Hary","age":21} -{"name":"Boo","age":22} -{"name":"Boo","age":21} -{"name":"Able","age":28} {"name":"Able","age":23} +{"name":"Able","age":28} +{"name":"Boo","age":21} +{"name":"Boo","age":22} +{"name":"Hary","age":21} +{"name":"Hary","age":87} PREHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW explode(sort_array_by(a_struct_array, 'name','age', 'DESC')) structTable AS a_struct PREHOOK: type: QUERY PREHOOK: Input: default@sort_array_by_table_order @@ -361,9 +361,9 @@ POSTHOOK: query: SELECT a_struct FROM sort_array_by_table_order LATERAL VIEW exp POSTHOOK: type: QUERY POSTHOOK: Input: default@sort_array_by_table_order #### A masked pattern was here #### -{"name":"Hary","age":87} -{"name":"Hary","age":21} -{"name":"Boo","age":22} -{"name":"Boo","age":21} -{"name":"Able","age":28} {"name":"Able","age":23} +{"name":"Able","age":28} +{"name":"Boo","age":21} +{"name":"Boo","age":22} +{"name":"Hary","age":21} +{"name":"Hary","age":87} diff --git ql/src/test/results/clientpositive/udf_soundex.q.out ql/src/test/results/clientpositive/llap/udf_soundex.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_soundex.q.out rename to ql/src/test/results/clientpositive/llap/udf_soundex.q.out index e9a560e1f3..5d248eabbf 100644 --- ql/src/test/results/clientpositive/udf_soundex.q.out +++ ql/src/test/results/clientpositive/llap/udf_soundex.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'M460' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_space.q.out ql/src/test/results/clientpositive/llap/udf_space.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_space.q.out rename to ql/src/test/results/clientpositive/llap/udf_space.q.out index f58b9c304a..d12c317e1d 100644 --- ql/src/test/results/clientpositive/udf_space.q.out +++ ql/src/test/results/clientpositive/llap/udf_space.q.out @@ -44,11 +44,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ' ' (type: string), '' (type: string), ' ' (type: string), '' (type: string), '' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 215500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_split.q.out ql/src/test/results/clientpositive/llap/udf_split.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_split.q.out rename to ql/src/test/results/clientpositive/llap/udf_split.q.out index c596fce8df..9188208406 100644 --- ql/src/test/results/clientpositive/udf_split.q.out +++ ql/src/test/results/clientpositive/llap/udf_split.q.out @@ -42,11 +42,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: split('a b c', ' ') (type: array), split('oneAtwoBthreeC', '[ABC]') (type: array), split('', '.') (type: array), split(50401020, 0) (type: array) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 1276000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_struct.q.out ql/src/test/results/clientpositive/llap/udf_struct.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_struct.q.out rename to ql/src/test/results/clientpositive/llap/udf_struct.q.out index df54a25047..2f7e2a609a 100644 --- ql/src/test/results/clientpositive/udf_struct.q.out +++ ql/src/test/results/clientpositive/llap/udf_struct.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: const struct(1) (type: struct), const struct(1,'a') (type: struct), struct(1,'b',1.5).col1 (type: int), struct(1,struct('a',1.5)).col2.col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT struct(1), struct(1, "a"), struct(1, "b", 1.5).col1, struct(1, struct("a", 1.5)).col2.col1 diff --git ql/src/test/results/clientpositive/udf_substring_index.q.out ql/src/test/results/clientpositive/llap/udf_substring_index.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_substring_index.q.out rename to ql/src/test/results/clientpositive/llap/udf_substring_index.q.out index e6b0fd7518..7b4bcb7891 100644 --- ql/src/test/results/clientpositive/udf_substring_index.q.out +++ ql/src/test/results/clientpositive/llap/udf_substring_index.q.out @@ -33,11 +33,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'www.apache' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_to_unix_timestamp.q.out ql/src/test/results/clientpositive/llap/udf_to_unix_timestamp.q.out similarity index 88% rename from ql/src/test/results/clientpositive/udf_to_unix_timestamp.q.out rename to ql/src/test/results/clientpositive/llap/udf_to_unix_timestamp.q.out index 2feb6dd738..626a0c9570 100644 --- ql/src/test/results/clientpositive/udf_to_unix_timestamp.q.out +++ ql/src/test/results/clientpositive/llap/udf_to_unix_timestamp.q.out @@ -106,14 +106,11 @@ STAGE PLANS: TableScan alias: src filterExpr: (to_unix_timestamp(key) > 10L) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (to_unix_timestamp(key) > 10L) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select * from (select * from src) a where to_unix_timestamp(a.key) > 10 @@ -135,13 +132,10 @@ STAGE PLANS: TableScan alias: src filterExpr: (to_unix_timestamp(key) > 10L) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (to_unix_timestamp(key) > 10L) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git ql/src/test/results/clientpositive/udf_to_utc_timestamp.q.out ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_to_utc_timestamp.q.out rename to ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out index 2aece90a20..62b73a36ac 100644 --- ql/src/test/results/clientpositive/udf_to_utc_timestamp.q.out +++ ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out @@ -29,11 +29,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: TIMESTAMP'2012-02-11 18:30:00' (type: timestamp) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select diff --git ql/src/test/results/clientpositive/udf_trunc.q.out ql/src/test/results/clientpositive/llap/udf_trunc.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_trunc.q.out rename to ql/src/test/results/clientpositive/llap/udf_trunc.q.out index 587ee80926..ea9686d782 100644 --- ql/src/test/results/clientpositive/udf_trunc.q.out +++ ql/src/test/results/clientpositive/llap/udf_trunc.q.out @@ -84,11 +84,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-02-01' (type: string), '2014-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-02-01' (type: string), '2014-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 1504 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -197,11 +195,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-04-01' (type: string), '2016-07-01' (type: string), '2016-10-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-04-01' (type: string), '2016-07-01' (type: string), '2016-10-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 - Statistics: Num rows: 1 Data size: 2068 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -310,11 +306,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 1504 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -411,11 +405,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-02-01' (type: string), '2014-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-02-01' (type: string), '2014-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 1504 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -518,11 +510,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-04-01' (type: string), '2016-07-01' (type: string), '2016-10-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18 - Statistics: Num rows: 1 Data size: 1786 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -625,11 +615,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 1504 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -710,11 +698,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-02-01' (type: string), '2014-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string), '2016-02-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -785,11 +771,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-04-01' (type: string), '2016-07-01' (type: string), '2016-10-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -860,11 +844,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2014-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string), '2016-01-01' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -925,11 +907,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-02-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 514 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -986,11 +966,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), null (type: string), null (type: string), null (type: string), '2014-01-01' (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 524 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT @@ -1047,11 +1025,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '2014-01-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 514 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT diff --git ql/src/test/results/clientpositive/udf_trunc_number.q.out ql/src/test/results/clientpositive/llap/udf_trunc_number.q.out similarity index 91% rename from ql/src/test/results/clientpositive/udf_trunc_number.q.out rename to ql/src/test/results/clientpositive/llap/udf_trunc_number.q.out index 6ff0444d20..8e05eb1c56 100644 --- ql/src/test/results/clientpositive/udf_trunc_number.q.out +++ ql/src/test/results/clientpositive/llap/udf_trunc_number.q.out @@ -17,11 +17,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1234567891.1234 (type: decimal(38,18)), 1234560000 (type: decimal(38,18)), 1234567891 (type: decimal(38,18)), 1234567891 (type: decimal(38,18)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 224000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT trunc(1234567891.1234567891,4), trunc(1234567891.1234567891,-4), trunc(1234567891.1234567891,0), trunc(1234567891.1234567891) FROM src tablesample (1 rows) @@ -77,11 +75,9 @@ STAGE PLANS: Processor Tree: TableScan alias: sampletable - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: trunc(c, d) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select trunc (c,d) from sampletable @@ -136,11 +132,9 @@ STAGE PLANS: Processor Tree: TableScan alias: sampletable1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: trunc(c, d) (type: float) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select trunc (c,d) from sampletable1 @@ -195,11 +189,9 @@ STAGE PLANS: Processor Tree: TableScan alias: sampletable2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: trunc(c, d) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select trunc (c,d) from sampletable2 @@ -254,11 +246,9 @@ STAGE PLANS: Processor Tree: TableScan alias: sampletable3 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: trunc(c, d) (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select trunc (c,d) from sampletable3 @@ -313,11 +303,9 @@ STAGE PLANS: Processor Tree: TableScan alias: sampletable4 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: trunc(c, d) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select trunc (c,d) from sampletable4 diff --git ql/src/test/results/clientpositive/udf_union.q.out ql/src/test/results/clientpositive/llap/udf_union.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_union.q.out rename to ql/src/test/results/clientpositive/llap/udf_union.q.out index 9aa4ec99a5..a2dc70d67f 100644 --- ql/src/test/results/clientpositive/udf_union.q.out +++ ql/src/test/results/clientpositive/llap/udf_union.q.out @@ -38,11 +38,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: create_union(0,key) (type: uniontype), create_union(if((key < 100), 0, 1),2,value) (type: uniontype), create_union(1,'a',const struct(2,'b')) (type: uniontype>) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 409000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT create_union(0, key), create_union(if(key<100, 0, 1), 2.0, value), diff --git ql/src/test/results/clientpositive/udf_when.q.out ql/src/test/results/clientpositive/llap/udf_when.q.out similarity index 95% rename from ql/src/test/results/clientpositive/udf_when.q.out rename to ql/src/test/results/clientpositive/llap/udf_when.q.out index ceb1e3cca6..8adf9e9c2a 100644 --- ql/src/test/results/clientpositive/udf_when.q.out +++ ql/src/test/results/clientpositive/llap/udf_when.q.out @@ -93,11 +93,9 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int), 9 (type: int), 14 (type: int), null (type: int), 24 (type: int), null (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 8008 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT CASE diff --git ql/src/test/results/clientpositive/udf_width_bucket.q.out ql/src/test/results/clientpositive/llap/udf_width_bucket.q.out similarity index 99% rename from ql/src/test/results/clientpositive/udf_width_bucket.q.out rename to ql/src/test/results/clientpositive/llap/udf_width_bucket.q.out index 4e7542e4c3..2db7c2a703 100644 --- ql/src/test/results/clientpositive/udf_width_bucket.q.out +++ ql/src/test/results/clientpositive/llap/udf_width_bucket.q.out @@ -36,11 +36,9 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 2 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select @@ -336,10 +334,10 @@ POSTHOOK: Input: default@alltypes_n3 #### A masked pattern was here #### 1 1 1 1 1 1 1 1 1 1 1 1 +11 11 11 11 11 11 3 3 3 3 3 3 7 7 7 7 7 7 8 8 8 8 8 8 -11 11 11 11 11 11 PREHOOK: query: truncate table alltypes_n3 PREHOOK: type: TRUNCATETABLE PREHOOK: Output: default@alltypes_n3 diff --git ql/src/test/results/clientpositive/llap/udtf_explode.q.out ql/src/test/results/clientpositive/llap/udtf_explode.q.out new file mode 100644 index 0000000000..95522c0df4 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udtf_explode.q.out @@ -0,0 +1,578 @@ +PREHOOK: query: DESCRIBE FUNCTION explode +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION explode +POSTHOOK: type: DESCFUNCTION +explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns +PREHOOK: query: DESCRIBE FUNCTION EXTENDED explode +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED explode +POSTHOOK: type: DESCFUNCTION +explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns +Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode +Function type:BUILTIN +PREHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 3 + Processor Tree: + TableScan + alias: src + GatherStats: false + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + UDTF Operator + function name: explode + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Limit + Number of rows: 3 + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Select Operator + expressions: col (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + null sort order: + numBuckets: -1 + sort order: + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + TopN: 3 + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int) + auto parallelism: false + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: int) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types int:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT explode(array(1, 2, 3)) AS myCol FROM src ORDER BY myCol LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(array(1, 2, 3)) AS myCol FROM src ORDER BY myCol LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 +1 +1 +PREHOOK: query: SELECT explode(array(1, 2, 3)) AS (myCol) FROM src ORDER BY myCol LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(array(1, 2, 3)) AS (myCol) FROM src ORDER BY myCol LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 +1 +1 +PREHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol ORDER BY a.myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1, 2, 3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol ORDER BY a.myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 1 +2 1 +3 1 +PREHOOK: query: EXPLAIN SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: map(1:'one',2:'two',3:'three') (type: map) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Top N Key Operator + sort order: ++ + keys: key (type: int), value (type: string) + null sort order: zz + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + top n: 3 + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 3 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: map(1:'one',2:'two',3:'three') (type: map) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + null sort order: + numBuckets: -1 + sort order: + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + TopN: 3 + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: string) + auto parallelism: false + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: int), _col1 (type: string) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + numBuckets: -1 + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col2 (type: bigint) + auto parallelism: true + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + numBuckets: -1 + sort order: ++ + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col2 (type: bigint) + auto parallelism: false + Reducer 4 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2 + columns.types int:string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src ORDER BY myKey, myVal LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 one +1 one +1 one +PREHOOK: query: SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1, 'one', 2, 'two', 3, 'three')) as (myKey, myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal ORDER BY a.myKey, a.myVal +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 one 1 +2 two 1 +3 three 1 +PREHOOK: query: SELECT src.key, myCol FROM src lateral view explode(array(1, 2, 3)) x AS myCol ORDER BY src.key, myCol LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT src.key, myCol FROM src lateral view explode(array(1, 2, 3)) x AS myCol ORDER BY src.key, myCol LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 1 +0 1 +0 1 +PREHOOK: query: SELECT src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 1 one +0 1 one +0 1 one +PREHOOK: query: SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1, 'one', 2, 'two', 3, 'three')) x AS myKey, myVal ORDER BY src.key, myKey, myVal LIMIT 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +968 0 1 one +2088 0 1 one +2632 0 1 one diff --git ql/src/test/results/clientpositive/udtf_get_sql_schema.q.out ql/src/test/results/clientpositive/llap/udtf_get_sql_schema.q.out similarity index 89% rename from ql/src/test/results/clientpositive/udtf_get_sql_schema.q.out rename to ql/src/test/results/clientpositive/llap/udtf_get_sql_schema.q.out index 2faf5aad57..8d6da899e2 100644 --- ql/src/test/results/clientpositive/udtf_get_sql_schema.q.out +++ ql/src/test/results/clientpositive/llap/udtf_get_sql_schema.q.out @@ -53,18 +53,14 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'select * from t1' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE UDTF Operator - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE function name: get_sql_schema Select Operator expressions: col_name (type: binary), col_type (type: binary) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select get_sql_schema('select * from t1') @@ -126,18 +122,14 @@ STAGE PLANS: TableScan alias: _dummy_table Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'select * from t2' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE UDTF Operator - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE function name: get_sql_schema Select Operator expressions: col_name (type: binary), col_type (type: binary) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select get_sql_schema('select * from t2') diff --git ql/src/test/results/clientpositive/llap/udtf_json_tuple.q.out ql/src/test/results/clientpositive/llap/udtf_json_tuple.q.out new file mode 100644 index 0000000000..6b2d07265f --- /dev/null +++ ql/src/test/results/clientpositive/llap/udtf_json_tuple.q.out @@ -0,0 +1,501 @@ +PREHOOK: query: create table json_t (key string, jstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_t +POSTHOOK: query: create table json_t (key string, jstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_t +PREHOOK: query: insert overwrite table json_t +select * from ( + select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src tablesample (1 rows) + union all + select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src tablesample (1 rows) + union all + select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src tablesample (1 rows) + union all + select '4', cast(null as string) from src tablesample (1 rows) + union all + select '5', '{"f1": "", "f5": null}' from src tablesample (1 rows) + union all + select '6', '[invalid JSON string]' from src tablesample (1 rows) +) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@json_t +POSTHOOK: query: insert overwrite table json_t +select * from ( + select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src tablesample (1 rows) + union all + select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src tablesample (1 rows) + union all + select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src tablesample (1 rows) + union all + select '4', cast(null as string) from src tablesample (1 rows) + union all + select '5', '{"f1": "", "f5": null}' from src tablesample (1 rows) + union all + select '6', '[invalid JSON string]' from src tablesample (1 rows) +) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@json_t +POSTHOOK: Lineage: json_t.jstring EXPRESSION [] +POSTHOOK: Lineage: json_t.key EXPRESSION [] +PREHOOK: query: explain +select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 3179 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Select Operator + expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + function name: json_tuple + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +1 value1 value2 3 NULL 5.23 +2 value12 2 value3 4.01 NULL +3 value13 2 value33 value44 5.01 +4 NULL NULL NULL NULL NULL +5 NULL NULL NULL NULL +6 NULL NULL NULL NULL NULL +PREHOOK: query: explain +select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE + function name: json_tuple + Select Operator + expressions: c0 (type: string), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + null sort order: zzz + sort order: +++ + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: string), _col4 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### + NULL NULL NULL NULL +value1 value2 3 NULL 5.23 +value12 2 value3 4.01 NULL +value13 2 value33 value44 5.01 +NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL +PREHOOK: query: explain +select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 3179 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col6 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) + Select Operator + expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + function name: json_tuple + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col6 (type: string), _col9 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +1 value2 5.23 +2 2 NULL +3 2 5.01 +4 NULL NULL +5 NULL NULL +6 NULL NULL +PREHOOK: query: explain +select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 6 Data size: 2669 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col6 (type: string) + outputColumnNames: _col6 + Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col6 (type: string) + minReductionHashAggr: 0.9166667 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + function name: json_tuple + Filter Operator + predicate: c0 is not null (type: boolean) + Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col6 (type: string) + outputColumnNames: _col6 + Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col6 (type: string) + minReductionHashAggr: 0.9166667 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_t +#### A masked pattern was here #### +POSTHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_t +#### A masked pattern was here #### +2 2 +value2 1 +NULL 1 +PREHOOK: query: CREATE TABLE dest1_n65(c1 STRING) STORED AS RCFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n65 +POSTHOOK: query: CREATE TABLE dest1_n65(c1 STRING) STORED AS RCFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n65 +PREHOOK: query: INSERT OVERWRITE TABLE dest1_n65 SELECT '{"a":"b\nc"}' FROM src tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n65 +POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n65 SELECT '{"a":"b\nc"}' FROM src tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n65 +POSTHOOK: Lineage: dest1_n65.c1 SIMPLE [] +PREHOOK: query: SELECT * FROM dest1_n65 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n65 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM dest1_n65 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n65 +#### A masked pattern was here #### +{"a":"b +c"} +PREHOOK: query: SELECT json FROM dest1_n65 a LATERAL VIEW json_tuple(c1, 'a') b AS json +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n65 +#### A masked pattern was here #### +POSTHOOK: query: SELECT json FROM dest1_n65 a LATERAL VIEW json_tuple(c1, 'a') b AS json +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n65 +#### A masked pattern was here #### +b +c diff --git ql/src/test/results/clientpositive/llap/udtf_parse_url_tuple.q.out ql/src/test/results/clientpositive/llap/udtf_parse_url_tuple.q.out new file mode 100644 index 0000000000..986b00a831 --- /dev/null +++ ql/src/test/results/clientpositive/llap/udtf_parse_url_tuple.q.out @@ -0,0 +1,468 @@ +PREHOOK: query: create table url_t (key string, fullurl string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@url_t +POSTHOOK: query: create table url_t (key string, fullurl string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@url_t +PREHOOK: query: insert overwrite table url_t +select * from ( + select '1', 'http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1' from src tablesample (1 rows) + union all + select '2', 'https://www.socs.uts.edu.au:80/MosaicDocs-old/url-primer.html?k1=tps#chapter1' from src tablesample (1 rows) + union all + select '3', 'ftp://sites.google.com/a/example.com/site/page' from src tablesample (1 rows) + union all + select '4', cast(null as string) from src tablesample (1 rows) + union all + select '5', 'htttp://' from src tablesample (1 rows) + union all + select '6', '[invalid url string]' from src tablesample (1 rows) +) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@url_t +POSTHOOK: query: insert overwrite table url_t +select * from ( + select '1', 'http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1' from src tablesample (1 rows) + union all + select '2', 'https://www.socs.uts.edu.au:80/MosaicDocs-old/url-primer.html?k1=tps#chapter1' from src tablesample (1 rows) + union all + select '3', 'ftp://sites.google.com/a/example.com/site/page' from src tablesample (1 rows) + union all + select '4', cast(null as string) from src tablesample (1 rows) + union all + select '5', 'htttp://' from src tablesample (1 rows) + union all + select '6', '[invalid url string]' from src tablesample (1 rows) +) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@url_t +POSTHOOK: Lineage: url_t.fullurl EXPRESSION [] +POSTHOOK: Lineage: url_t.key EXPRESSION [] +PREHOOK: query: describe function parse_url_tuple +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: describe function parse_url_tuple +POSTHOOK: type: DESCFUNCTION +parse_url_tuple(url, partname1, partname2, ..., partnameN) - extracts N (N>=1) parts from a URL. +It takes a URL and one or multiple partnames, and returns a tuple. All the input parameters and output column types are string. +PREHOOK: query: describe function extended parse_url_tuple +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: describe function extended parse_url_tuple +POSTHOOK: type: DESCFUNCTION +parse_url_tuple(url, partname1, partname2, ..., partnameN) - extracts N (N>=1) parts from a URL. +It takes a URL and one or multiple partnames, and returns a tuple. All the input parameters and output column types are string. +Partname: HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, USERINFO, QUERY: +Note: Partnames are case-sensitive, and should not contain unnecessary white spaces. +Example: + > SELECT b.* FROM src LATERAL VIEW parse_url_tuple(fullurl, 'HOST', 'PATH', 'QUERY', 'QUERY:id') b as host, path, query, query_id LIMIT 1; + > SELECT parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from src a; +Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFParseUrlTuple +Function type:BUILTIN +PREHOOK: query: explain +select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 3159 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 12 Data size: 8598 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) + Select Operator + expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE + function name: parse_url_tuple + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 12 Data size: 8598 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: string), VALUE._col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +1 facebook.com /path1/p.php k1=v1&k2=v2 Ref1 http /path1/p.php?k1=v1&k2=v2 facebook.com NULL v1 +2 www.socs.uts.edu.au /MosaicDocs-old/url-primer.html k1=tps chapter1 https /MosaicDocs-old/url-primer.html?k1=tps www.socs.uts.edu.au:80 NULL tps +3 sites.google.com /a/example.com/site/page NULL NULL ftp /a/example.com/site/page sites.google.com NULL NULL +4 NULL NULL NULL NULL NULL NULL NULL NULL NULL +5 NULL NULL NULL NULL NULL NULL NULL NULL NULL +6 NULL NULL NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: explain +select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 6 Data size: 5556 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 5556 Basic stats: COMPLETE Column stats: COMPLETE + function name: parse_url_tuple + Select Operator + expressions: c0 (type: string), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string), c5 (type: string), c6 (type: string), c7 (type: string), c8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + null sort order: zzz + sort order: +++ + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +NULL NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL NULL +NULL NULL NULL NULL NULL NULL NULL NULL NULL +facebook.com /path1/p.php k1=v1&k2=v2 Ref1 http /path1/p.php?k1=v1&k2=v2 facebook.com NULL v1 +sites.google.com /a/example.com/site/page NULL NULL ftp /a/example.com/site/page sites.google.com NULL NULL +www.socs.uts.edu.au /MosaicDocs-old/url-primer.html k1=tps chapter1 https /MosaicDocs-old/url-primer.html?k1=tps www.socs.uts.edu.au:80 NULL tps +PREHOOK: query: explain +select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 6 Data size: 3159 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 12 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col7 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + Select Operator + expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string), 'host' (type: string), 'query' (type: string), 'QUERY:nonExistCol' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 6 Data size: 7107 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 7107 Basic stats: COMPLETE Column stats: COMPLETE + function name: parse_url_tuple + Lateral View Join Operator + outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 + Statistics: Num rows: 12 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col7 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +1 facebook.com k1=v1&k2=v2 v1 NULL NULL NULL +2 www.socs.uts.edu.au k1=tps tps NULL NULL NULL +3 sites.google.com NULL NULL NULL NULL NULL +4 NULL NULL NULL NULL NULL NULL +5 NULL NULL NULL NULL NULL NULL +6 NULL NULL NULL NULL NULL NULL +PREHOOK: query: explain +select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: explain +select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 6 Data size: 2649 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: string) + outputColumnNames: _col5 + Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col5 (type: string) + minReductionHashAggr: 0.9166667 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE + function name: parse_url_tuple + Filter Operator + predicate: c8 is not null (type: boolean) + Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: string) + outputColumnNames: _col5 + Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col5 (type: string) + minReductionHashAggr: 0.9166667 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho +PREHOOK: type: QUERY +PREHOOK: Input: default@url_t +#### A masked pattern was here #### +POSTHOOK: query: select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho +POSTHOOK: type: QUERY +POSTHOOK: Input: default@url_t +#### A masked pattern was here #### +facebook.com 1 +www.socs.uts.edu.au 1 diff --git ql/src/test/results/clientpositive/llap/udtf_stack.q.out ql/src/test/results/clientpositive/llap/udtf_stack.q.out new file mode 100644 index 0000000000..34adbf9cbb --- /dev/null +++ ql/src/test/results/clientpositive/llap/udtf_stack.q.out @@ -0,0 +1,149 @@ +PREHOOK: query: DESCRIBE FUNCTION stack +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION stack +POSTHOOK: type: DESCFUNCTION +stack(n, cols...) - turns k columns into n rows of size k/n each +PREHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 2 + Processor Tree: + TableScan + alias: src + Lateral View Forward + Select Operator + Lateral View Join Operator + outputColumnNames: _col5, _col6 + Select Operator + expressions: _col5 (type: string), _col6 (type: array) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 2 + ListSink + Select Operator + expressions: 2 (type: int), 'x' (type: string), array(1) (type: array), 'z' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + UDTF Operator + function name: stack + Lateral View Join Operator + outputColumnNames: _col5, _col6 + Select Operator + expressions: _col5 (type: string), _col6 (type: array) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 2 + ListSink + +PREHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 2 + Processor Tree: + TableScan + alias: src + Lateral View Forward + Select Operator + Lateral View Join Operator + outputColumnNames: _col5, _col6 + Select Operator + expressions: _col5 (type: string), _col6 (type: array) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 2 + ListSink + Select Operator + expressions: 2 (type: int), 'x' (type: string), array(1) (type: array), 'z' (type: string), array(4) (type: array) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + UDTF Operator + function name: stack + Lateral View Join Operator + outputColumnNames: _col5, _col6 + Select Operator + expressions: _col5 (type: string), _col6 (type: array) + outputColumnNames: _col0, _col1 + Limit + Number of rows: 2 + ListSink + +PREHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +x [1] +z NULL +PREHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +x [1] +z [4] +PREHOOK: query: EXPLAIN +SELECT stack(1, "en", "dbpedia", NULL ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT stack(1, "en", "dbpedia", NULL ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Select Operator + expressions: 1 (type: int), 'en' (type: string), 'dbpedia' (type: string), null (type: void) + outputColumnNames: _col0, _col1, _col2, _col3 + UDTF Operator + function name: stack + Select Operator + expressions: col0 (type: string), col1 (type: string), null (type: void) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +en dbpedia NULL diff --git ql/src/test/results/clientpositive/unicode_comments.q.out ql/src/test/results/clientpositive/llap/unicode_comments.q.out similarity index 100% rename from ql/src/test/results/clientpositive/unicode_comments.q.out rename to ql/src/test/results/clientpositive/llap/unicode_comments.q.out diff --git ql/src/test/results/clientpositive/unicode_data.q.out ql/src/test/results/clientpositive/llap/unicode_data.q.out similarity index 75% rename from ql/src/test/results/clientpositive/unicode_data.q.out rename to ql/src/test/results/clientpositive/llap/unicode_data.q.out index 0149e5f8bc..7de2b73596 100644 --- ql/src/test/results/clientpositive/unicode_data.q.out +++ ql/src/test/results/clientpositive/llap/unicode_data.q.out @@ -69,40 +69,24 @@ POSTHOOK: Input: unicode_data_db@t_test POSTHOOK: Input: unicode_data_db@t_view_test #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t_test - filterExpr: (name = '李四') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (name = '李四') (type: boolean) - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: t_test + filterExpr: (name = '李四') (type: boolean) + properties: + insideView TRUE + Filter Operator + predicate: (name = '李四') (type: boolean) + Select Operator + expressions: value (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select * from t_view_test PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/union.q.out ql/src/test/results/clientpositive/llap/union.q.out similarity index 69% rename from ql/src/test/results/clientpositive/union.q.out rename to ql/src/test/results/clientpositive/llap/union.q.out index 9126189641..1b3036298e 100644 --- ql/src/test/results/clientpositive/union.q.out +++ ql/src/test/results/clientpositive/llap/union.q.out @@ -4,7 +4,7 @@ FROM ( UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: target/warehouse/union.out @@ -14,72 +14,84 @@ FROM ( UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: target/warehouse/union.out STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) < 100.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 100.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src - filterExpr: (UDFToDouble(key) > 100.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) > 100.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) < 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 100.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) > 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) > 100.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: Select Operator - expressions: key (type: string), value (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Union + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Conditional Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -87,40 +99,12 @@ STAGE PLANS: hdfs directory: true destination: target/warehouse/union.out - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM ( FROM src select src.key, src.value WHERE src.key < 100 UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: target/warehouse/union.out @@ -129,505 +113,505 @@ POSTHOOK: query: FROM ( UNION ALL FROM src SELECT src.* WHERE src.key > 100 ) unioninput -INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* +INSERT OVERWRITE DIRECTORY 'target/warehouse/union.out' SELECT unioninput.* ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: target/warehouse/union.out -238val_238 -86val_86 -311val_311 -27val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66val_66 +0val_0 +0val_0 +0val_0 +10val_10 +103val_103 +103val_103 +104val_104 +104val_104 +105val_105 +11val_11 +111val_111 +113val_113 +113val_113 +114val_114 +116val_116 +118val_118 +118val_118 +119val_119 +119val_119 +119val_119 +12val_12 +12val_12 +120val_120 +120val_120 +125val_125 +125val_125 +126val_126 128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 +128val_128 +128val_128 +129val_129 +129val_129 +131val_131 +133val_133 +134val_134 +134val_134 +136val_136 +137val_137 +137val_137 +138val_138 +138val_138 +138val_138 +138val_138 +143val_143 145val_145 -495val_495 -37val_37 -327val_327 -281val_281 -277val_277 -209val_209 +146val_146 +146val_146 +149val_149 +149val_149 15val_15 -82val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 +15val_15 +150val_150 +152val_152 +152val_152 153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 +155val_155 +156val_156 +157val_157 +158val_158 +160val_160 162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 +163val_163 +164val_164 +164val_164 +165val_165 +165val_165 +166val_166 167val_167 -195val_195 -475val_475 +167val_167 +167val_167 +168val_168 +169val_169 +169val_169 +169val_169 +169val_169 17val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 170val_170 -20val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92val_92 -111val_111 -47val_47 -72val_72 -4val_4 -280val_280 -35val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 +172val_172 +172val_172 +174val_174 +174val_174 +175val_175 +175val_175 176val_176 -54val_54 -459val_459 -51val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 176val_176 -289val_289 -221val_221 -65val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83val_83 -333val_333 +177val_177 +178val_178 +179val_179 +179val_179 +18val_18 +18val_18 180val_180 -284val_284 -12val_12 -230val_230 181val_181 -67val_67 -260val_260 -404val_404 -384val_384 -489val_489 -353val_353 -373val_373 -272val_272 -138val_138 -217val_217 -84val_84 -348val_348 -466val_466 -58val_58 -8val_8 -411val_411 -230val_230 -208val_208 -348val_348 -24val_24 -463val_463 -431val_431 -179val_179 -172val_172 -42val_42 -129val_129 -158val_158 -119val_119 -496val_496 -0val_0 -322val_322 -197val_197 -468val_468 -393val_393 -454val_454 -298val_298 -199val_199 +183val_183 +186val_186 +187val_187 +187val_187 +187val_187 +189val_189 +19val_19 +190val_190 191val_191 -418val_418 -96val_96 -26val_26 -165val_165 -327val_327 -230val_230 -205val_205 -120val_120 -131val_131 -51val_51 -404val_404 -43val_43 -436val_436 -156val_156 -469val_469 -468val_468 -308val_308 -95val_95 +191val_191 +192val_192 +193val_193 +193val_193 +193val_193 +194val_194 +195val_195 +195val_195 196val_196 -288val_288 -481val_481 -457val_457 -98val_98 -282val_282 197val_197 -187val_187 -318val_318 -318val_318 -409val_409 -470val_470 -137val_137 -369val_369 -316val_316 -169val_169 -413val_413 -85val_85 -77val_77 -0val_0 -490val_490 -87val_87 -364val_364 -179val_179 -118val_118 -134val_134 -395val_395 -282val_282 -138val_138 -238val_238 -419val_419 -15val_15 -118val_118 -72val_72 -90val_90 -307val_307 -19val_19 -435val_435 -10val_10 -277val_277 -273val_273 -306val_306 +197val_197 +199val_199 +199val_199 +199val_199 +2val_2 +20val_20 +200val_200 +200val_200 +201val_201 +202val_202 +203val_203 +203val_203 +205val_205 +205val_205 +207val_207 +207val_207 +208val_208 +208val_208 +208val_208 +209val_209 +209val_209 +213val_213 +213val_213 +214val_214 +216val_216 +216val_216 +217val_217 +217val_217 +218val_218 +219val_219 +219val_219 +221val_221 +221val_221 +222val_222 +223val_223 +223val_223 +224val_224 224val_224 -309val_309 -389val_389 -327val_327 -242val_242 -369val_369 -392val_392 -272val_272 -331val_331 -401val_401 -242val_242 -452val_452 -177val_177 226val_226 -5val_5 -497val_497 -402val_402 -396val_396 -317val_317 -395val_395 -58val_58 -35val_35 -336val_336 -95val_95 -11val_11 -168val_168 -34val_34 -229val_229 -233val_233 -143val_143 -472val_472 -322val_322 -498val_498 -160val_160 -195val_195 -42val_42 -321val_321 -430val_430 -119val_119 -489val_489 -458val_458 -78val_78 -76val_76 -41val_41 -223val_223 -492val_492 -149val_149 -449val_449 -218val_218 228val_228 -138val_138 -453val_453 -30val_30 -209val_209 -64val_64 -468val_468 -76val_76 -74val_74 -342val_342 -69val_69 +229val_229 +229val_229 230val_230 -33val_33 -368val_368 -103val_103 -296val_296 -113val_113 -216val_216 -367val_367 -344val_344 -167val_167 -274val_274 -219val_219 +230val_230 +230val_230 +230val_230 +230val_230 +233val_233 +233val_233 +235val_235 +237val_237 +237val_237 +238val_238 +238val_238 239val_239 -485val_485 -116val_116 -223val_223 +239val_239 +24val_24 +24val_24 +241val_241 +242val_242 +242val_242 +244val_244 +247val_247 +248val_248 +249val_249 +252val_252 +255val_255 +255val_255 +256val_256 256val_256 +257val_257 +258val_258 +26val_26 +26val_26 +260val_260 +262val_262 263val_263 -70val_70 -487val_487 -480val_480 -401val_401 -288val_288 -191val_191 -5val_5 -244val_244 -438val_438 -128val_128 -467val_467 -432val_432 -202val_202 -316val_316 -229val_229 -469val_469 -463val_463 +265val_265 +265val_265 +266val_266 +27val_27 +272val_272 +272val_272 +273val_273 +273val_273 +273val_273 +274val_274 +275val_275 +277val_277 +277val_277 +277val_277 +277val_277 +278val_278 +278val_278 +28val_28 280val_280 -2val_2 -35val_35 +280val_280 +281val_281 +281val_281 +282val_282 +282val_282 283val_283 -331val_331 -235val_235 -80val_80 -44val_44 -193val_193 +284val_284 +285val_285 +286val_286 +287val_287 +288val_288 +288val_288 +289val_289 +291val_291 +292val_292 +296val_296 +298val_298 +298val_298 +298val_298 +30val_30 +302val_302 +305val_305 +306val_306 +307val_307 +307val_307 +308val_308 +309val_309 +309val_309 +310val_310 +311val_311 +311val_311 +311val_311 +315val_315 +316val_316 +316val_316 +316val_316 +317val_317 +317val_317 +318val_318 +318val_318 +318val_318 321val_321 +321val_321 +322val_322 +322val_322 +323val_323 +325val_325 +325val_325 +327val_327 +327val_327 +327val_327 +33val_33 +331val_331 +331val_331 +332val_332 +333val_333 +333val_333 335val_335 -104val_104 -466val_466 +336val_336 +338val_338 +339val_339 +34val_34 +341val_341 +342val_342 +342val_342 +344val_344 +344val_344 +345val_345 +348val_348 +348val_348 +348val_348 +348val_348 +348val_348 +35val_35 +35val_35 +35val_35 +351val_351 +353val_353 +353val_353 +356val_356 +360val_360 +362val_362 +364val_364 +365val_365 366val_366 -175val_175 +367val_367 +367val_367 +368val_368 +369val_369 +369val_369 +369val_369 +37val_37 +37val_37 +373val_373 +374val_374 +375val_375 +377val_377 +378val_378 +379val_379 +382val_382 +382val_382 +384val_384 +384val_384 +384val_384 +386val_386 +389val_389 +392val_392 +393val_393 +394val_394 +395val_395 +395val_395 +396val_396 +396val_396 +396val_396 +397val_397 +397val_397 +399val_399 +399val_399 +4val_4 +400val_400 +401val_401 +401val_401 +401val_401 +401val_401 +401val_401 +402val_402 403val_403 -483val_483 -53val_53 -105val_105 -257val_257 +403val_403 +403val_403 +404val_404 +404val_404 406val_406 +406val_406 +406val_406 +406val_406 +407val_407 +409val_409 +409val_409 409val_409 -190val_190 -406val_406 -401val_401 -114val_114 -258val_258 -90val_90 -203val_203 -262val_262 -348val_348 +41val_41 +411val_411 +413val_413 +413val_413 +414val_414 +414val_414 +417val_417 +417val_417 +417val_417 +418val_418 +419val_419 +42val_42 +42val_42 +421val_421 424val_424 -12val_12 -396val_396 -201val_201 -217val_217 -164val_164 +424val_424 +427val_427 +429val_429 +429val_429 +43val_43 +430val_430 +430val_430 +430val_430 431val_431 -454val_454 -478val_478 -298val_298 -125val_125 431val_431 -164val_164 -424val_424 -187val_187 -382val_382 -5val_5 -70val_70 -397val_397 -480val_480 -291val_291 -24val_24 -351val_351 -255val_255 -104val_104 -70val_70 -163val_163 +431val_431 +432val_432 +435val_435 +436val_436 +437val_437 +438val_438 +438val_438 438val_438 -119val_119 -414val_414 -200val_200 -491val_491 -237val_237 439val_439 -360val_360 -248val_248 -479val_479 -305val_305 -417val_417 -199val_199 -444val_444 -120val_120 -429val_429 -169val_169 +439val_439 +44val_44 443val_443 -323val_323 -325val_325 -277val_277 -230val_230 -478val_478 -178val_178 -468val_468 -310val_310 -317val_317 -333val_333 -493val_493 +444val_444 +446val_446 +448val_448 +449val_449 +452val_452 +453val_453 +454val_454 +454val_454 +454val_454 +455val_455 +457val_457 +458val_458 +458val_458 +459val_459 +459val_459 460val_460 -207val_207 -249val_249 -265val_265 -480val_480 -83val_83 -136val_136 -353val_353 -172val_172 -214val_214 462val_462 -233val_233 -406val_406 -133val_133 -175val_175 -189val_189 -454val_454 -375val_375 -401val_401 -421val_421 -407val_407 -384val_384 -256val_256 -26val_26 -134val_134 -67val_67 -384val_384 -379val_379 -18val_18 462val_462 +463val_463 +463val_463 +466val_466 +466val_466 +466val_466 +467val_467 +468val_468 +468val_468 +468val_468 +468val_468 +469val_469 +469val_469 +469val_469 +469val_469 +469val_469 +47val_47 +470val_470 +472val_472 +475val_475 +477val_477 +478val_478 +478val_478 +479val_479 +480val_480 +480val_480 +480val_480 +481val_481 +482val_482 +483val_483 +484val_484 +485val_485 +487val_487 +489val_489 +489val_489 +489val_489 +489val_489 +490val_490 +491val_491 492val_492 -298val_298 -9val_9 -341val_341 +492val_492 +493val_493 +494val_494 +495val_495 +496val_496 +497val_497 498val_498 -146val_146 -458val_458 -362val_362 -186val_186 -285val_285 -348val_348 -167val_167 -18val_18 -273val_273 -183val_183 -281val_281 -344val_344 -97val_97 -469val_469 -315val_315 +498val_498 +498val_498 +5val_5 +5val_5 +5val_5 +51val_51 +51val_51 +53val_53 +54val_54 +57val_57 +58val_58 +58val_58 +64val_64 +65val_65 +66val_66 +67val_67 +67val_67 +69val_69 +70val_70 +70val_70 +70val_70 +72val_72 +72val_72 +74val_74 +76val_76 +76val_76 +77val_77 +78val_78 +8val_8 +80val_80 +82val_82 +83val_83 +83val_83 84val_84 -28val_28 -37val_37 -448val_448 -152val_152 -348val_348 -307val_307 -194val_194 -414val_414 -477val_477 -222val_222 -126val_126 +84val_84 +85val_85 +86val_86 +87val_87 +9val_9 90val_90 -169val_169 -403val_403 -400val_400 -200val_200 +90val_90 +90val_90 +92val_92 +95val_95 +95val_95 +96val_96 +97val_97 97val_97 +98val_98 +98val_98 diff --git ql/src/test/results/clientpositive/llap/union10.q.out ql/src/test/results/clientpositive/llap/union10.q.out new file mode 100644 index 0000000000..d139bea3a8 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union10.q.out @@ -0,0 +1,292 @@ +PREHOOK: query: create table tmptable_n3(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable_n3 +POSTHOOK: query: create table tmptable_n3(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable_n3 +PREHOOK: query: explain +insert overwrite table tmptable_n3 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable_n3 +POSTHOOK: query: explain +insert overwrite table tmptable_n3 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n3 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n3 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n3 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, int + Table: default.tmptable_n3 + +PREHOOK: query: insert overwrite table tmptable_n3 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable_n3 +POSTHOOK: query: insert overwrite table tmptable_n3 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable_n3 +POSTHOOK: Lineage: tmptable_n3.key EXPRESSION [] +POSTHOOK: Lineage: tmptable_n3.value EXPRESSION [(src)s1.null, (src)s2.null, (src)s3.null, ] +PREHOOK: query: select * from tmptable_n3 x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable_n3 +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable_n3 x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable_n3 +#### A masked pattern was here #### +tst1 500 +tst2 500 +tst3 500 diff --git ql/src/test/results/clientpositive/llap/union11.q.out ql/src/test/results/clientpositive/llap/union11.q.out new file mode 100644 index 0000000000..9fd590c567 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union11.q.out @@ -0,0 +1,221 @@ +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +tst3 1 diff --git ql/src/test/results/clientpositive/llap/union12.q.out ql/src/test/results/clientpositive/llap/union12.q.out new file mode 100644 index 0000000000..480848068a --- /dev/null +++ ql/src/test/results/clientpositive/llap/union12.q.out @@ -0,0 +1,300 @@ +PREHOOK: query: create table tmptable_n10(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable_n10 +POSTHOOK: query: create table tmptable_n10(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable_n10 +PREHOOK: query: explain +insert overwrite table tmptable_n10 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src1 s2 + UNION ALL + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@tmptable_n10 +POSTHOOK: query: explain +insert overwrite table tmptable_n10 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src1 s2 + UNION ALL + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@tmptable_n10 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.96 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n10 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n10 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n10 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, value + Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n10 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, int + Table: default.tmptable_n10 + +PREHOOK: query: insert overwrite table tmptable_n10 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src1 s2 + UNION ALL + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@tmptable_n10 +POSTHOOK: query: insert overwrite table tmptable_n10 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src1 s2 + UNION ALL + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@tmptable_n10 +POSTHOOK: Lineage: tmptable_n10.key EXPRESSION [] +POSTHOOK: Lineage: tmptable_n10.value EXPRESSION [(src)s1.null, (src1)s2.null, (srcbucket)s3.null, ] +PREHOOK: query: select * from tmptable_n10 x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable_n10 +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable_n10 x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable_n10 +#### A masked pattern was here #### +tst1 500 +tst2 25 +tst3 1000 diff --git ql/src/test/results/clientpositive/union13.q.out ql/src/test/results/clientpositive/llap/union13.q.out similarity index 86% rename from ql/src/test/results/clientpositive/union13.q.out rename to ql/src/test/results/clientpositive/llap/union13.q.out index 159de366cf..688732e6de 100644 --- ql/src/test/results/clientpositive/union13.q.out +++ ql/src/test/results/clientpositive/llap/union13.q.out @@ -16,40 +16,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -67,1003 +78,1003 @@ POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -238 val_238 -238 val_238 -86 val_86 -86 val_86 -311 val_311 -311 val_311 -27 val_27 -27 val_27 -165 val_165 -165 val_165 -409 val_409 -409 val_409 -255 val_255 -255 val_255 -278 val_278 -278 val_278 -98 val_98 -98 val_98 -484 val_484 -484 val_484 -265 val_265 -265 val_265 -193 val_193 -193 val_193 -401 val_401 -401 val_401 -150 val_150 -150 val_150 -273 val_273 -273 val_273 -224 val_224 -224 val_224 -369 val_369 -369 val_369 -66 val_66 -66 val_66 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +10 val_10 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +105 val_105 +105 val_105 +11 val_11 +11 val_11 +111 val_111 +111 val_111 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +114 val_114 +114 val_114 +116 val_116 +116 val_116 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +126 val_126 +126 val_126 128 val_128 128 val_128 -213 val_213 -213 val_213 -146 val_146 -146 val_146 -406 val_406 -406 val_406 -429 val_429 -429 val_429 -374 val_374 -374 val_374 -152 val_152 -152 val_152 -469 val_469 -469 val_469 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +131 val_131 +131 val_131 +133 val_133 +133 val_133 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +136 val_136 +136 val_136 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +143 val_143 145 val_145 145 val_145 -495 val_495 -495 val_495 -37 val_37 -37 val_37 -327 val_327 -327 val_327 -281 val_281 -281 val_281 -277 val_277 -277 val_277 -209 val_209 -209 val_209 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +149 val_149 +149 val_149 15 val_15 15 val_15 -82 val_82 -82 val_82 -403 val_403 -403 val_403 -166 val_166 -166 val_166 -417 val_417 -417 val_417 -430 val_430 -430 val_430 -252 val_252 -252 val_252 -292 val_292 -292 val_292 -219 val_219 -219 val_219 -287 val_287 -287 val_287 +15 val_15 +15 val_15 +150 val_150 +150 val_150 +152 val_152 +152 val_152 +152 val_152 +152 val_152 153 val_153 153 val_153 -193 val_193 -193 val_193 -338 val_338 -338 val_338 -446 val_446 -446 val_446 -459 val_459 -459 val_459 -394 val_394 -394 val_394 -237 val_237 -237 val_237 -482 val_482 -482 val_482 -174 val_174 -174 val_174 -413 val_413 -413 val_413 -494 val_494 -494 val_494 -207 val_207 -207 val_207 -199 val_199 -199 val_199 -466 val_466 -466 val_466 -208 val_208 -208 val_208 -174 val_174 -174 val_174 -399 val_399 -399 val_399 -396 val_396 -396 val_396 -247 val_247 -247 val_247 -417 val_417 -417 val_417 -489 val_489 -489 val_489 +155 val_155 +155 val_155 +156 val_156 +156 val_156 +157 val_157 +157 val_157 +158 val_158 +158 val_158 +160 val_160 +160 val_160 162 val_162 162 val_162 -377 val_377 -377 val_377 -397 val_397 -397 val_397 -309 val_309 -309 val_309 -365 val_365 -365 val_365 -266 val_266 -266 val_266 -439 val_439 -439 val_439 -342 val_342 -342 val_342 -367 val_367 -367 val_367 -325 val_325 -325 val_325 +163 val_163 +163 val_163 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +166 val_166 +166 val_166 167 val_167 167 val_167 -195 val_195 -195 val_195 -475 val_475 -475 val_475 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 17 val_17 17 val_17 -113 val_113 -113 val_113 -155 val_155 -155 val_155 -203 val_203 -203 val_203 -339 val_339 -339 val_339 -0 val_0 -0 val_0 -455 val_455 -455 val_455 -128 val_128 -128 val_128 -311 val_311 -311 val_311 -316 val_316 -316 val_316 -57 val_57 -57 val_57 -302 val_302 -302 val_302 -205 val_205 -205 val_205 -149 val_149 -149 val_149 -438 val_438 -438 val_438 -345 val_345 -345 val_345 -129 val_129 -129 val_129 170 val_170 170 val_170 -20 val_20 -20 val_20 -489 val_489 -489 val_489 -157 val_157 -157 val_157 -378 val_378 -378 val_378 -221 val_221 -221 val_221 -92 val_92 -92 val_92 -111 val_111 -111 val_111 -47 val_47 -47 val_47 -72 val_72 -72 val_72 -4 val_4 -4 val_4 -280 val_280 -280 val_280 -35 val_35 -35 val_35 -427 val_427 -427 val_427 -277 val_277 -277 val_277 -208 val_208 -208 val_208 -356 val_356 -356 val_356 -399 val_399 -399 val_399 -169 val_169 -169 val_169 -382 val_382 -382 val_382 -498 val_498 -498 val_498 -125 val_125 -125 val_125 -386 val_386 -386 val_386 -437 val_437 -437 val_437 -469 val_469 -469 val_469 -192 val_192 -192 val_192 -286 val_286 -286 val_286 -187 val_187 -187 val_187 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +175 val_175 +175 val_175 176 val_176 176 val_176 -54 val_54 -54 val_54 -459 val_459 -459 val_459 -51 val_51 -51 val_51 -138 val_138 -138 val_138 -103 val_103 -103 val_103 -239 val_239 -239 val_239 -213 val_213 -213 val_213 -216 val_216 -216 val_216 -430 val_430 -430 val_430 -278 val_278 -278 val_278 176 val_176 176 val_176 -289 val_289 -289 val_289 -221 val_221 -221 val_221 -65 val_65 -65 val_65 -318 val_318 -318 val_318 -332 val_332 -332 val_332 -311 val_311 -311 val_311 -275 val_275 -275 val_275 -137 val_137 -137 val_137 -241 val_241 -241 val_241 -83 val_83 -83 val_83 -333 val_333 -333 val_333 +177 val_177 +177 val_177 +178 val_178 +178 val_178 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +18 val_18 +18 val_18 180 val_180 180 val_180 -284 val_284 -284 val_284 -12 val_12 -12 val_12 -230 val_230 -230 val_230 181 val_181 181 val_181 -67 val_67 -67 val_67 -260 val_260 -260 val_260 -404 val_404 -404 val_404 -384 val_384 -384 val_384 -489 val_489 -489 val_489 -353 val_353 -353 val_353 -373 val_373 -373 val_373 -272 val_272 -272 val_272 -138 val_138 -138 val_138 -217 val_217 -217 val_217 -84 val_84 -84 val_84 -348 val_348 -348 val_348 -466 val_466 -466 val_466 -58 val_58 -58 val_58 -8 val_8 -8 val_8 -411 val_411 -411 val_411 -230 val_230 -230 val_230 -208 val_208 -208 val_208 -348 val_348 -348 val_348 -24 val_24 -24 val_24 -463 val_463 -463 val_463 -431 val_431 -431 val_431 -179 val_179 -179 val_179 -172 val_172 -172 val_172 -42 val_42 -42 val_42 -129 val_129 -129 val_129 -158 val_158 -158 val_158 -119 val_119 -119 val_119 -496 val_496 -496 val_496 -0 val_0 -0 val_0 -322 val_322 -322 val_322 -197 val_197 -197 val_197 -468 val_468 -468 val_468 -393 val_393 -393 val_393 -454 val_454 -454 val_454 -100 val_100 -100 val_100 -298 val_298 -298 val_298 -199 val_199 -199 val_199 -191 val_191 -191 val_191 -418 val_418 -418 val_418 -96 val_96 -96 val_96 -26 val_26 -26 val_26 -165 val_165 -165 val_165 -327 val_327 -327 val_327 -230 val_230 -230 val_230 -205 val_205 -205 val_205 -120 val_120 -120 val_120 -131 val_131 -131 val_131 -51 val_51 -51 val_51 -404 val_404 -404 val_404 -43 val_43 -43 val_43 -436 val_436 -436 val_436 -156 val_156 -156 val_156 -469 val_469 -469 val_469 -468 val_468 -468 val_468 -308 val_308 -308 val_308 -95 val_95 -95 val_95 -196 val_196 -196 val_196 -288 val_288 -288 val_288 -481 val_481 -481 val_481 -457 val_457 -457 val_457 -98 val_98 -98 val_98 -282 val_282 -282 val_282 -197 val_197 -197 val_197 +183 val_183 +183 val_183 +186 val_186 +186 val_186 187 val_187 187 val_187 -318 val_318 -318 val_318 -318 val_318 -318 val_318 -409 val_409 -409 val_409 -470 val_470 -470 val_470 -137 val_137 -137 val_137 -369 val_369 -369 val_369 -316 val_316 -316 val_316 -169 val_169 -169 val_169 -413 val_413 -413 val_413 -85 val_85 -85 val_85 -77 val_77 -77 val_77 -0 val_0 -0 val_0 -490 val_490 -490 val_490 -87 val_87 -87 val_87 -364 val_364 -364 val_364 -179 val_179 -179 val_179 -118 val_118 -118 val_118 -134 val_134 -134 val_134 -395 val_395 -395 val_395 -282 val_282 -282 val_282 -138 val_138 -138 val_138 -238 val_238 -238 val_238 -419 val_419 -419 val_419 -15 val_15 -15 val_15 -118 val_118 -118 val_118 -72 val_72 -72 val_72 -90 val_90 -90 val_90 -307 val_307 -307 val_307 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +189 val_189 19 val_19 19 val_19 -435 val_435 -435 val_435 -10 val_10 -10 val_10 -277 val_277 -277 val_277 -273 val_273 -273 val_273 -306 val_306 -306 val_306 -224 val_224 -224 val_224 -309 val_309 -309 val_309 -389 val_389 -389 val_389 -327 val_327 -327 val_327 -242 val_242 -242 val_242 -369 val_369 -369 val_369 -392 val_392 -392 val_392 -272 val_272 -272 val_272 -331 val_331 -331 val_331 -401 val_401 -401 val_401 -242 val_242 -242 val_242 -452 val_452 -452 val_452 -177 val_177 -177 val_177 -226 val_226 -226 val_226 -5 val_5 -5 val_5 -497 val_497 -497 val_497 -402 val_402 -402 val_402 -396 val_396 -396 val_396 -317 val_317 -317 val_317 -395 val_395 -395 val_395 -58 val_58 -58 val_58 -35 val_35 -35 val_35 -336 val_336 -336 val_336 -95 val_95 -95 val_95 -11 val_11 -11 val_11 -168 val_168 -168 val_168 -34 val_34 -34 val_34 -229 val_229 -229 val_229 -233 val_233 -233 val_233 -143 val_143 -143 val_143 -472 val_472 -472 val_472 -322 val_322 -322 val_322 -498 val_498 -498 val_498 -160 val_160 -160 val_160 +190 val_190 +190 val_190 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +192 val_192 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +194 val_194 195 val_195 195 val_195 -42 val_42 -42 val_42 -321 val_321 -321 val_321 -430 val_430 -430 val_430 -119 val_119 -119 val_119 -489 val_489 -489 val_489 -458 val_458 -458 val_458 -78 val_78 -78 val_78 -76 val_76 -76 val_76 -41 val_41 -41 val_41 -223 val_223 -223 val_223 -492 val_492 -492 val_492 -149 val_149 -149 val_149 -449 val_449 -449 val_449 -218 val_218 -218 val_218 -228 val_228 -228 val_228 -138 val_138 -138 val_138 -453 val_453 -453 val_453 -30 val_30 -30 val_30 +195 val_195 +195 val_195 +196 val_196 +196 val_196 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +2 val_2 +2 val_2 +20 val_20 +20 val_20 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +201 val_201 +201 val_201 +202 val_202 +202 val_202 +203 val_203 +203 val_203 +203 val_203 +203 val_203 +205 val_205 +205 val_205 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +207 val_207 +207 val_207 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 209 val_209 209 val_209 -64 val_64 -64 val_64 -468 val_468 -468 val_468 -76 val_76 -76 val_76 -74 val_74 -74 val_74 -342 val_342 -342 val_342 -69 val_69 -69 val_69 -230 val_230 -230 val_230 -33 val_33 -33 val_33 -368 val_368 -368 val_368 -103 val_103 -103 val_103 -296 val_296 -296 val_296 -113 val_113 -113 val_113 +209 val_209 +209 val_209 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +214 val_214 +214 val_214 216 val_216 216 val_216 -367 val_367 -367 val_367 -344 val_344 -344 val_344 -167 val_167 -167 val_167 -274 val_274 -274 val_274 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +218 val_218 +218 val_218 219 val_219 219 val_219 -239 val_239 -239 val_239 -485 val_485 -485 val_485 -116 val_116 -116 val_116 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +222 val_222 +222 val_222 223 val_223 223 val_223 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +226 val_226 +226 val_226 +228 val_228 +228 val_228 +229 val_229 +229 val_229 +229 val_229 +229 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +235 val_235 +235 val_235 +237 val_237 +237 val_237 +237 val_237 +237 val_237 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +241 val_241 +241 val_241 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +244 val_244 +244 val_244 +247 val_247 +247 val_247 +248 val_248 +248 val_248 +249 val_249 +249 val_249 +252 val_252 +252 val_252 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +256 val_256 +256 val_256 256 val_256 256 val_256 +257 val_257 +257 val_257 +258 val_258 +258 val_258 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +260 val_260 +260 val_260 +262 val_262 +262 val_262 263 val_263 263 val_263 -70 val_70 -70 val_70 -487 val_487 -487 val_487 -480 val_480 -480 val_480 -401 val_401 -401 val_401 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +266 val_266 +266 val_266 +27 val_27 +27 val_27 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +274 val_274 +275 val_275 +275 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +28 val_28 +28 val_28 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +283 val_283 +283 val_283 +284 val_284 +284 val_284 +285 val_285 +285 val_285 +286 val_286 +286 val_286 +287 val_287 +287 val_287 288 val_288 288 val_288 -191 val_191 -191 val_191 -5 val_5 -5 val_5 -244 val_244 -244 val_244 -438 val_438 -438 val_438 -128 val_128 -128 val_128 -467 val_467 -467 val_467 -432 val_432 -432 val_432 -202 val_202 -202 val_202 +288 val_288 +288 val_288 +289 val_289 +289 val_289 +291 val_291 +291 val_291 +292 val_292 +292 val_292 +296 val_296 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +30 val_30 +302 val_302 +302 val_302 +305 val_305 +305 val_305 +306 val_306 +306 val_306 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +308 val_308 +308 val_308 +309 val_309 +309 val_309 +309 val_309 +309 val_309 +310 val_310 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +315 val_315 316 val_316 316 val_316 -229 val_229 -229 val_229 -469 val_469 -469 val_469 -463 val_463 -463 val_463 -280 val_280 -280 val_280 -2 val_2 -2 val_2 -35 val_35 -35 val_35 -283 val_283 -283 val_283 -331 val_331 -331 val_331 -235 val_235 -235 val_235 -80 val_80 -80 val_80 -44 val_44 -44 val_44 -193 val_193 -193 val_193 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 321 val_321 321 val_321 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +323 val_323 +323 val_323 +325 val_325 +325 val_325 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +33 val_33 +331 val_331 +331 val_331 +331 val_331 +331 val_331 +332 val_332 +332 val_332 +333 val_333 +333 val_333 +333 val_333 +333 val_333 335 val_335 335 val_335 -104 val_104 -104 val_104 -466 val_466 -466 val_466 +336 val_336 +336 val_336 +338 val_338 +338 val_338 +339 val_339 +339 val_339 +34 val_34 +34 val_34 +341 val_341 +341 val_341 +342 val_342 +342 val_342 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +345 val_345 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +351 val_351 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +356 val_356 +356 val_356 +360 val_360 +360 val_360 +362 val_362 +362 val_362 +364 val_364 +364 val_364 +365 val_365 +365 val_365 366 val_366 366 val_366 -175 val_175 -175 val_175 +367 val_367 +367 val_367 +367 val_367 +367 val_367 +368 val_368 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +373 val_373 +373 val_373 +374 val_374 +374 val_374 +375 val_375 +375 val_375 +377 val_377 +377 val_377 +378 val_378 +378 val_378 +379 val_379 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +386 val_386 +389 val_389 +389 val_389 +392 val_392 +392 val_392 +393 val_393 +393 val_393 +394 val_394 +394 val_394 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +4 val_4 +4 val_4 +400 val_400 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 +402 val_402 403 val_403 403 val_403 -483 val_483 -483 val_483 -53 val_53 -53 val_53 -105 val_105 -105 val_105 -257 val_257 -257 val_257 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +404 val_404 +404 val_404 406 val_406 406 val_406 -409 val_409 -409 val_409 -190 val_190 -190 val_190 406 val_406 406 val_406 -401 val_401 -401 val_401 -114 val_114 -114 val_114 -258 val_258 -258 val_258 -90 val_90 -90 val_90 -203 val_203 -203 val_203 -262 val_262 -262 val_262 -348 val_348 -348 val_348 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +41 val_41 +411 val_411 +411 val_411 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +418 val_418 +419 val_419 +419 val_419 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +421 val_421 +421 val_421 424 val_424 424 val_424 -12 val_12 -12 val_12 -396 val_396 -396 val_396 -201 val_201 -201 val_201 -217 val_217 -217 val_217 -164 val_164 -164 val_164 +424 val_424 +424 val_424 +427 val_427 +427 val_427 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +43 val_43 +43 val_43 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 431 val_431 431 val_431 -454 val_454 -454 val_454 -478 val_478 -478 val_478 -298 val_298 -298 val_298 -125 val_125 -125 val_125 431 val_431 431 val_431 -164 val_164 -164 val_164 -424 val_424 -424 val_424 -187 val_187 -187 val_187 -382 val_382 -382 val_382 -5 val_5 -5 val_5 -70 val_70 -70 val_70 -397 val_397 -397 val_397 -480 val_480 -480 val_480 -291 val_291 -291 val_291 -24 val_24 -24 val_24 -351 val_351 -351 val_351 -255 val_255 -255 val_255 -104 val_104 -104 val_104 -70 val_70 -70 val_70 -163 val_163 -163 val_163 +431 val_431 +431 val_431 +432 val_432 +432 val_432 +435 val_435 +435 val_435 +436 val_436 +436 val_436 +437 val_437 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +438 val_438 438 val_438 438 val_438 -119 val_119 -119 val_119 -414 val_414 -414 val_414 -200 val_200 -200 val_200 -491 val_491 -491 val_491 -237 val_237 -237 val_237 439 val_439 439 val_439 -360 val_360 -360 val_360 -248 val_248 -248 val_248 -479 val_479 -479 val_479 -305 val_305 -305 val_305 -417 val_417 -417 val_417 -199 val_199 -199 val_199 -444 val_444 -444 val_444 -120 val_120 -120 val_120 -429 val_429 -429 val_429 -169 val_169 -169 val_169 +439 val_439 +439 val_439 +44 val_44 +44 val_44 443 val_443 443 val_443 -323 val_323 -323 val_323 -325 val_325 -325 val_325 -277 val_277 -277 val_277 -230 val_230 -230 val_230 -478 val_478 -478 val_478 -178 val_178 -178 val_178 -468 val_468 -468 val_468 -310 val_310 -310 val_310 -317 val_317 -317 val_317 -333 val_333 -333 val_333 -493 val_493 -493 val_493 -460 val_460 -460 val_460 -207 val_207 -207 val_207 -249 val_249 -249 val_249 -265 val_265 -265 val_265 -480 val_480 -480 val_480 -83 val_83 -83 val_83 -136 val_136 -136 val_136 -353 val_353 -353 val_353 -172 val_172 -172 val_172 -214 val_214 -214 val_214 -462 val_462 -462 val_462 -233 val_233 -233 val_233 -406 val_406 -406 val_406 -133 val_133 -133 val_133 -175 val_175 -175 val_175 -189 val_189 -189 val_189 +444 val_444 +444 val_444 +446 val_446 +446 val_446 +448 val_448 +448 val_448 +449 val_449 +449 val_449 +452 val_452 +452 val_452 +453 val_453 +453 val_453 454 val_454 454 val_454 -375 val_375 -375 val_375 -401 val_401 -401 val_401 -421 val_421 -421 val_421 -407 val_407 -407 val_407 -384 val_384 -384 val_384 -256 val_256 -256 val_256 -26 val_26 -26 val_26 -134 val_134 -134 val_134 -67 val_67 -67 val_67 -384 val_384 -384 val_384 -379 val_379 -379 val_379 -18 val_18 -18 val_18 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +455 val_455 +457 val_457 +457 val_457 +458 val_458 +458 val_458 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +460 val_460 +460 val_460 462 val_462 462 val_462 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +47 val_47 +470 val_470 +470 val_470 +472 val_472 +472 val_472 +475 val_475 +475 val_475 +477 val_477 +477 val_477 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +479 val_479 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +481 val_481 +482 val_482 +482 val_482 +483 val_483 +483 val_483 +484 val_484 +484 val_484 +485 val_485 +485 val_485 +487 val_487 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +490 val_490 +491 val_491 +491 val_491 492 val_492 492 val_492 -100 val_100 -100 val_100 -298 val_298 -298 val_298 -9 val_9 -9 val_9 -341 val_341 -341 val_341 +492 val_492 +492 val_492 +493 val_493 +493 val_493 +494 val_494 +494 val_494 +495 val_495 +495 val_495 +496 val_496 +496 val_496 +497 val_497 +497 val_497 498 val_498 498 val_498 -146 val_146 -146 val_146 -458 val_458 -458 val_458 -362 val_362 -362 val_362 -186 val_186 -186 val_186 -285 val_285 -285 val_285 -348 val_348 -348 val_348 -167 val_167 -167 val_167 -18 val_18 -18 val_18 -273 val_273 -273 val_273 -183 val_183 -183 val_183 -281 val_281 -281 val_281 -344 val_344 -344 val_344 -97 val_97 -97 val_97 -469 val_469 -469 val_469 -315 val_315 -315 val_315 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +53 val_53 +54 val_54 +54 val_54 +57 val_57 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +64 val_64 +65 val_65 +65 val_65 +66 val_66 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +77 val_77 +78 val_78 +78 val_78 +8 val_8 +8 val_8 +80 val_80 +80 val_80 +82 val_82 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 84 val_84 84 val_84 -28 val_28 -28 val_28 -37 val_37 -37 val_37 -448 val_448 -448 val_448 -152 val_152 -152 val_152 -348 val_348 -348 val_348 -307 val_307 -307 val_307 -194 val_194 -194 val_194 -414 val_414 -414 val_414 -477 val_477 -477 val_477 -222 val_222 -222 val_222 -126 val_126 -126 val_126 +84 val_84 +84 val_84 +85 val_85 +85 val_85 +86 val_86 +86 val_86 +87 val_87 +87 val_87 +9 val_9 +9 val_9 90 val_90 90 val_90 -169 val_169 -169 val_169 -403 val_403 -403 val_403 -400 val_400 -400 val_400 -200 val_200 -200 val_200 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +96 val_96 +97 val_97 +97 val_97 97 val_97 97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/llap/union14.q.out ql/src/test/results/clientpositive/llap/union14.q.out new file mode 100644 index 0000000000..5f45c392a4 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union14.q.out @@ -0,0 +1,163 @@ +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 10 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 diff --git ql/src/test/results/clientpositive/llap/union15.q.out ql/src/test/results/clientpositive/llap/union15.q.out new file mode 100644 index 0000000000..41bcb6e363 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union15.q.out @@ -0,0 +1,193 @@ +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6862745 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6862745 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.6862745 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2 + UNION ALL + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 20 +128 2 +146 2 +150 2 +213 2 +224 2 +238 2 +255 2 +273 2 +278 2 +311 2 +369 2 +401 2 +406 2 +66 2 +98 2 +tst1 1 diff --git ql/src/test/results/clientpositive/llap/union16.q.out ql/src/test/results/clientpositive/llap/union16.q.out new file mode 100644 index 0000000000..0d36dcc1fe --- /dev/null +++ ql/src/test/results/clientpositive/llap/union16.q.out @@ -0,0 +1,746 @@ +PREHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 10 <- Union 2 (CONTAINS) + Map 11 <- Union 2 (CONTAINS) + Map 12 <- Union 2 (CONTAINS) + Map 13 <- Union 2 (CONTAINS) + Map 14 <- Union 2 (CONTAINS) + Map 15 <- Union 2 (CONTAINS) + Map 16 <- Union 2 (CONTAINS) + Map 17 <- Union 2 (CONTAINS) + Map 18 <- Union 2 (CONTAINS) + Map 19 <- Union 2 (CONTAINS) + Map 20 <- Union 2 (CONTAINS) + Map 21 <- Union 2 (CONTAINS) + Map 22 <- Union 2 (CONTAINS) + Map 23 <- Union 2 (CONTAINS) + Map 24 <- Union 2 (CONTAINS) + Map 25 <- Union 2 (CONTAINS) + Map 26 <- Union 2 (CONTAINS) + Map 27 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Union 2 (CONTAINS) + Map 8 <- Union 2 (CONTAINS) + Map 9 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 10 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 12 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 14 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 15 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 16 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 17 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 18 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 19 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 20 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 21 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 22 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 23 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 24 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 25 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 26 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 27 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src UNION ALL + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +12500 diff --git ql/src/test/results/clientpositive/union17.q.out ql/src/test/results/clientpositive/llap/union17.q.out similarity index 54% rename from ql/src/test/results/clientpositive/union17.q.out rename to ql/src/test/results/clientpositive/llap/union17.q.out index 45b0862965..1b32388013 100644 --- ql/src/test/results/clientpositive/union17.q.out +++ ql/src/test/results/clientpositive/llap/union17.q.out @@ -38,153 +38,225 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0, Stage-5, Stage-8 - Stage-5 depends on stages: Stage-3 - Stage-7 depends on stages: Stage-1, Stage-5, Stage-8 - Stage-6 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-6 - Stage-8 depends on stages: Stage-6 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Map 8 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Union 3 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + null sort order: zzz + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + null sort order: zzz + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE + aggregations: count(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n78 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - minReductionHashAggr: 0.99 - mode: hash + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(DISTINCT KEY._col2:0._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 501 Data size: 140280 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), CAST( _col2 AS STRING) (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Reduce Operator Tree: - Group By Operator - aggregations: count(DISTINCT KEY._col1:0._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n78 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2_n17 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, val1, val2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 Move Operator @@ -204,85 +276,6 @@ STAGE PLANS: Column Types: string, string Table: default.dest1_n78 - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, val1, val2 - Column Types: string, string, string - Table: default.dest2_n17 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - null sort order: zzz - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 501 Data size: 229959 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(DISTINCT KEY._col2:0._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 140280 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2_n17 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, val1, val2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 Move Operator tables: @@ -293,29 +286,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2_n17 - Stage: Stage-8 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, val1, val2 + Column Types: string, string, string + Table: default.dest2_n17 PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/union18.q.out ql/src/test/results/clientpositive/llap/union18.q.out similarity index 70% rename from ql/src/test/results/clientpositive/union18.q.out rename to ql/src/test/results/clientpositive/llap/union18.q.out index 841d40fffe..2538051311 100644 --- ql/src/test/results/clientpositive/union18.q.out +++ ql/src/test/results/clientpositive/llap/union18.q.out @@ -37,194 +37,200 @@ POSTHOOK: Output: default@dest2_n33 STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 - Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 - Stage-6 - Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 - Stage-4 depends on stages: Stage-0, Stage-11 - Stage-5 - Stage-7 - Stage-8 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-10 depends on stages: Stage-1, Stage-11 - Stage-11 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n128 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 501 Data size: 136272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2_n33 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, val1, val2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Union 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n128 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 501 Data size: 136272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n128 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 501 Data size: 136272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2_n33 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, val1, val2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2_n33 + name: default.dest1_n128 Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, val1, val2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 501 Data size: 136272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-9 - Conditional Operator + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2_n33 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, val1, val2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-3 + Dependency Collection Stage: Stage-0 Move Operator @@ -244,36 +250,6 @@ STAGE PLANS: Column Types: string, string Table: default.dest1_n128 - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n128 - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n128 - - Stage: Stage-8 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - Stage: Stage-1 Move Operator tables: @@ -284,7 +260,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2_n33 - Stage: Stage-10 + Stage: Stage-5 Stats Work Basic Stats Work: Column Stats Desc: @@ -292,30 +268,6 @@ STAGE PLANS: Column Types: string, string, string Table: default.dest2_n33 - Stage: Stage-11 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc diff --git ql/src/test/results/clientpositive/union19.q.out ql/src/test/results/clientpositive/llap/union19.q.out similarity index 67% rename from ql/src/test/results/clientpositive/union19.q.out rename to ql/src/test/results/clientpositive/llap/union19.q.out index 1907b9024e..966a053370 100644 --- ql/src/test/results/clientpositive/union19.q.out +++ ql/src/test/results/clientpositive/llap/union19.q.out @@ -38,184 +38,218 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0, Stage-5, Stage-7 - Stage-5 depends on stages: Stage-3 - Stage-6 depends on stages: Stage-1, Stage-5, Stage-7 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-7 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: string) + minReductionHashAggr: 0.500998 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2_n21 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, val1, val2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2_n21 Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, val1, val2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') - minReductionHashAggr: 0.99 + aggregations: count(_col1) + keys: _col0 (type: string) + minReductionHashAggr: 0.500998 mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 501 Data size: 89272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2_n21 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, val1, val2 + Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2_n21 + name: default.dest1_n86 Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, val1, val2 - Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(val1, 'hll'), compute_stats(val2, 'hll') + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n86 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection Stage: Stage-0 Move Operator @@ -235,38 +269,6 @@ STAGE PLANS: Column Types: string, string Table: default.dest1_n86 - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, val1, val2 - Column Types: string, string, string - Table: default.dest2_n21 - Stage: Stage-1 Move Operator tables: @@ -277,29 +279,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2_n21 - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, val1, val2 + Column Types: string, string, string + Table: default.dest2_n21 PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/llap/union20.q.out ql/src/test/results/clientpositive/llap/union20.q.out new file mode 100644 index 0000000000..bf7985080d --- /dev/null +++ ql/src/test/results/clientpositive/llap/union20.q.out @@ -0,0 +1,223 @@ +PREHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION ALL + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION ALL + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 3 (CONTAINS) + Map 8 <- Union 6 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 6 (SIMPLE_EDGE) + Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s2 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s4 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 168 Data size: 91392 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 168 Data size: 91392 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Union 3 + Vertex: Union 3 + Union 6 + Vertex: Union 6 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION ALL + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION ALL + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +tst1 500 tst1 500 diff --git ql/src/test/results/clientpositive/llap/union21.q.out ql/src/test/results/clientpositive/llap/union21.q.out new file mode 100644 index 0000000000..13f30b023a --- /dev/null +++ ql/src/test/results/clientpositive/llap/union21.q.out @@ -0,0 +1,773 @@ +PREHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION ALL + SELECT reverse(key) as key from src + UNION ALL + SELECT key as key from src + UNION ALL + SELECT astring as key from src_thrift + UNION ALL + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION ALL + SELECT reverse(key) as key from src + UNION ALL + SELECT key as key from src + UNION ALL + SELECT astring as key from src_thrift + UNION ALL + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: reverse(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: astring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: lstring[0] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION ALL + SELECT reverse(key) as key from src + UNION ALL + SELECT key as key from src + UNION ALL + SELECT astring as key from src_thrift + UNION ALL + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION ALL + SELECT reverse(key) as key from src + UNION ALL + SELECT key as key from src + UNION ALL + SELECT astring as key from src_thrift + UNION ALL + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +0 7 +001 2 +002 2 +004 1 +01 1 +013 1 +02 1 +021 2 +03 1 +032 5 +034 3 +051 1 +061 1 +062 1 +063 1 +064 1 +07 3 +071 1 +074 1 +08 1 +081 1 +082 2 +084 3 +09 3 +091 1 +094 1 +1 500 +10 2 +100 2 +102 1 +103 2 +104 7 +105 1 +11 2 +111 2 +113 5 +114 2 +116 1 +118 2 +119 3 +12 2 +120 2 +122 2 +123 2 +124 1 +125 2 +126 1 +128 3 +129 2 +131 2 +133 3 +134 5 +136 1 +137 2 +138 4 +14 1 +142 1 +143 2 +145 1 +146 2 +149 2 +15 4 +150 1 +152 2 +153 2 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 2 +165 2 +166 1 +167 3 +168 1 +169 4 +17 1 +170 1 +172 2 +174 2 +175 2 +176 2 +177 1 +178 1 +179 2 +18 2 +180 1 +181 2 +182 2 +183 1 +184 1 +186 1 +187 3 +189 1 +19 1 +190 1 +191 4 +192 2 +193 3 +194 2 +195 2 +196 1 +197 2 +199 3 +2 2 +20 2 +200 2 +201 1 +202 2 +203 3 +204 1 +205 2 +207 2 +208 3 +209 2 +21 2 +213 2 +214 1 +216 2 +217 2 +218 1 +219 2 +221 2 +222 2 +223 4 +224 2 +226 1 +228 1 +229 2 +230 5 +233 3 +234 1 +235 1 +237 2 +238 2 +239 2 +24 4 +241 1 +242 4 +243 2 +244 1 +247 1 +248 1 +249 1 +251 2 +252 2 +254 1 +255 2 +256 2 +257 1 +258 1 +26 2 +260 1 +261 1 +262 2 +263 2 +264 2 +265 2 +266 1 +27 3 +271 2 +272 4 +273 3 +274 2 +275 1 +277 4 +278 2 +28 2 +280 2 +281 2 +282 4 +283 3 +284 2 +285 1 +286 1 +287 1 +288 2 +289 1 +29 1 +291 2 +292 2 +293 1 +294 2 +296 1 +298 3 +30 2 +301 2 +302 3 +304 3 +305 1 +306 1 +307 2 +308 1 +309 2 +310 1 +311 5 +312 2 +314 2 +315 1 +316 3 +317 2 +318 3 +321 2 +322 4 +323 2 +325 2 +327 3 +33 2 +331 3 +332 3 +333 4 +335 1 +336 1 +338 1 +339 1 +34 2 +341 2 +342 2 +344 3 +345 1 +348 5 +35 4 +351 2 +353 4 +354 1 +356 1 +360 1 +361 1 +362 2 +364 3 +365 1 +366 1 +367 2 +368 1 +369 3 +37 2 +372 3 +373 2 +374 1 +375 1 +377 1 +378 1 +379 1 +38 2 +381 1 +382 3 +384 4 +386 1 +389 1 +391 3 +392 1 +393 2 +394 2 +395 2 +396 3 +397 2 +399 2 +4 2 +40 1 +400 1 +401 7 +402 1 +403 3 +404 4 +406 4 +407 1 +409 3 +41 1 +411 2 +412 1 +413 2 +414 4 +417 3 +418 1 +419 1 +42 4 +421 1 +422 2 +424 4 +427 1 +429 2 +43 2 +430 3 +431 5 +432 1 +435 1 +436 1 +437 1 +438 3 +439 2 +44 2 +442 1 +443 3 +444 2 +446 1 +448 1 +449 1 +45 1 +452 1 +453 1 +454 6 +455 1 +457 1 +458 2 +459 2 +46 1 +460 1 +461 2 +462 2 +463 3 +466 3 +467 1 +468 4 +469 5 +47 2 +470 1 +471 2 +472 2 +473 1 +475 1 +477 1 +478 2 +479 1 +48 2 +480 3 +481 1 +482 2 +483 4 +484 2 +485 1 +487 1 +489 4 +490 1 +491 2 +492 2 +493 2 +494 2 +495 1 +496 1 +497 1 +498 3 +5 6 +50 1 +501 1 +502 2 +503 1 +51 4 +513 1 +521 2 +523 2 +53 4 +532 1 +533 1 +534 1 +54 1 +541 1 +543 1 +551 1 +552 2 +554 1 +56 1 +561 2 +562 2 +563 1 +57 1 +571 2 +572 1 +573 1 +574 1 +58 3 +582 1 +584 1 +59 2 +591 2 +593 2 +594 1 +60 1 +603 1 +604 4 +611 1 +612 2 +613 3 +62 2 +621 1 +622 1 +631 1 +633 1 +634 1 +64 1 +641 2 +644 1 +65 1 +651 1 +652 2 +653 1 +66 2 +661 1 +662 1 +663 1 +664 3 +67 4 +671 2 +68 1 +681 1 +682 1 +683 1 +69 2 +691 1 +692 1 +693 3 +694 1 +70 4 +702 2 +703 2 +704 1 +71 1 +712 2 +713 2 +714 3 +72 3 +723 3 +724 1 +73 2 +731 2 +732 2 +734 1 +74 2 +742 1 +75 1 +751 1 +752 1 +754 1 +76 4 +761 3 +763 2 +764 1 +77 2 +771 1 +772 4 +773 1 +774 1 +78 2 +781 3 +782 1 +784 1 +79 2 +791 2 +793 2 +794 1 +8 2 +80 2 +802 3 +803 1 +81 2 +811 2 +812 1 +813 3 +814 1 +82 2 +821 3 +822 1 +83 2 +831 4 +832 2 +833 1 +834 3 +84 2 +842 1 +843 5 +844 1 +85 3 +851 1 +852 1 +854 2 +86 1 +861 1 +863 1 +864 4 +87 2 +871 1 +872 2 +873 1 +874 2 +882 2 +89 2 +892 3 +894 3 +9 2 +90 4 +902 2 +903 2 +904 3 +91 1 +911 3 +912 2 +914 1 +92 1 +921 2 +922 2 +924 2 +932 2 +933 1 +934 2 +941 2 +942 1 +944 1 +95 2 +954 2 +96 2 +961 4 +963 3 +964 5 +97 2 +971 2 +973 1 +974 1 +98 2 +981 1 +982 1 +983 1 +984 4 +991 3 +993 2 +NULL 2 +record_0 1 +record_1 1 +record_2 1 +record_3 1 +record_4 1 +record_5 1 +record_6 1 +record_7 1 +record_8 1 +record_9 1 diff --git ql/src/test/results/clientpositive/union22.q.out ql/src/test/results/clientpositive/llap/union22.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union22.q.out rename to ql/src/test/results/clientpositive/llap/union22.q.out index de36e44dfb..d6cf8f78c4 100644 --- ql/src/test/results/clientpositive/union22.q.out +++ ql/src/test/results/clientpositive/llap/union22.q.out @@ -92,25 +92,96 @@ LEFT JOIN (SELECT `k1`, `k3`, `k4` FROM `default`.`dst_union22_delta` WHERE `k0` > 50 AND `k1` > 20 AND `ds` = '1') AS `t4` ON `t2`.`k1` = `t4`.`k1` AND `t2`.`=` STAGE DEPENDENCIES: - Stage-7 is a root stage , consists of Stage-8, Stage-4 - Stage-8 has a backup stage: Stage-4 - Stage-6 depends on stages: Stage-8 - Stage-2 depends on stages: Stage-4, Stage-6 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 STAGE PLANS: - Stage: Stage-7 - Conditional Operator - - Stage: Stage-8 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0-subquery2:$hdt$_1:dst_union22_delta - Fetch Operator - limit: -1 - Partition Description: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dst_union22_delta + filterExpr: ((UDFToDouble(k0) <= 50.0D) and (ds = '1')) (type: boolean) + Statistics: Num rows: 500 Data size: 221500 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(k0) <= 50.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 73538 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 166 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), '2' (type: string) + outputColumnNames: k1, k2, k3, k4, ds + Statistics: Num rows: 387 Data size: 141297 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(k1, 'hll'), compute_stats(k2, 'hll'), compute_stats(k3, 'hll'), compute_stats(k4, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat @@ -158,251 +229,168 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dst_union22_delta name: default.dst_union22_delta - Alias -> Map Local Operator Tree: - $hdt$_0-subquery2:$hdt$_1:dst_union22_delta - TableScan - alias: dst_union22_delta - filterExpr: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) - Statistics: Num rows: 500 Data size: 176000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) - Statistics: Num rows: 55 Data size: 19360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: k1 (type: string), k3 (type: string), k4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 55 Data size: 14575 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter mappings: - 0 [1, 1] - filter predicates: - 0 {_col2} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Position of Big Table: 0 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: (UDFToDouble(k1) > 20.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(k1) > 20.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 60092 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: k1 (type: string), k2 (type: string), (ds = '1') (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 30212 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - filter mappings: - 0 [1, 1] - filter predicates: - 0 {_col2} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Position of Big Table: 0 - Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Execution mode: vectorized - Local Work: - Map Reduce Local Work - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k1":"true","k2":"true","k3":"true","k4":"true"}} - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 11124 - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11624 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Map 4 + Map Operator Tree: + TableScan + alias: a + filterExpr: (UDFToDouble(k1) > 20.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(k1) > 20.0D) (type: boolean) + Statistics: Num rows: 166 Data size: 60092 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: k1 (type: string), k2 (type: string), (ds = '1') (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 166 Data size: 30212 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + filter mappings: + 0 [1, 1] + filter predicates: + 0 {_col2} + 1 + Estimated key counts: Map 5 => 55 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col4, _col5 + input vertices: + 1 Map 5 + Position of Big Table: 0 + Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - name: default.dst_union22 + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k0":"true","k1":"true","k2":"true","k3":"true","k4":"true","k5":"true"}} - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string #### A masked pattern was here #### - name default.dst_union22_delta - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 16936 - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17436 + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), '2' (type: string) + outputColumnNames: k1, k2, k3, k4, ds + Statistics: Num rows: 387 Data size: 141297 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(k1, 'hll'), compute_stats(k2, 'hll'), compute_stats(k3, 'hll'), compute_stats(k4, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: #### A masked pattern was here #### - name default.dst_union22_delta - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Path -> Partition: #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22_delta - name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22/ds=1 [$hdt$_0-subquery2:$hdt$_0:a] - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: dst_union22_delta - filterExpr: ((UDFToDouble(k0) <= 50.0D) and (ds = '1')) (type: boolean) - Statistics: Num rows: 500 Data size: 221500 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(k0) <= 50.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 73538 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 1 + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k1":"true","k2":"true","k3":"true","k4":"true"}} + bucket_count -1 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2/ - Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE + name default.dst_union22 + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 11124 + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11624 #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string #### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), '2' (type: string) - outputColumnNames: k1, k2, k3, k4, ds - Statistics: Num rows: 387 Data size: 141297 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(k1, 'hll'), compute_stats(k2, 'hll'), compute_stats(k3, 'hll'), compute_stats(k4, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + name: default.dst_union22 + Truncated Path -> Alias: + /dst_union22/ds=1 [a] + Map 5 + Map Operator Tree: + TableScan + alias: dst_union22_delta + filterExpr: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 176000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 19360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: k1 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 14575 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator bucketingVersion: 2 key expressions: _col0 (type: string) @@ -410,180 +398,108 @@ STAGE PLANS: numBuckets: -1 sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) - auto parallelism: false - TableScan - GatherStats: false - Union - Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 1 + Statistics: Num rows: 55 Data size: 14575 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: _col1 (type: string), _col2 (type: string) + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k0":"true","k1":"true","k2":"true","k3":"true","k4":"true","k5":"true"}} + bucket_count -1 + column.name.delimiter , + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2/ - Statistics: Num rows: 387 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 #### A masked pattern was here #### - table: + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 bucketing_version 2 column.name.delimiter , - columns k1,k2,k3,k4 + columns k0,k1,k2,k3,k4,k5 columns.comments - columns.types string:string:string:string + columns.types string:string:string:string:string:string #### A masked pattern was here #### - name default.dst_union22 + name default.dst_union22_delta partition_columns ds partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), '2' (type: string) - outputColumnNames: k1, k2, k3, k4, ds - Statistics: Num rows: 387 Data size: 141297 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(k1, 'hll'), compute_stats(k2, 'hll'), compute_stats(k3, 'hll'), compute_stats(k4, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Reducer 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + File Output Operator bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10002 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k0":"true","k1":"true","k2":"true","k3":"true","k4":"true","k5":"true"}} - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 16936 - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17436 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22_delta - name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22_delta/ds=1 [$hdt$_0-subquery1:dst_union22_delta] -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 1845 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2,_col3,_col4 - columns.types struct:struct:struct:struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2,_col3,_col4 + columns.types struct:struct:struct:struct:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -623,198 +539,6 @@ STAGE PLANS: Table: default.dst_union22 Is Table Level Stats: false - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: (UDFToDouble(k1) > 20.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(k1) > 20.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 60092 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: k1 (type: string), k2 (type: string), (ds = '1') (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 30212 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 30212 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: _col1 (type: string), _col2 (type: boolean) - auto parallelism: false - TableScan - alias: dst_union22_delta - filterExpr: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) - Statistics: Num rows: 500 Data size: 176000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((UDFToDouble(k0) > 50.0D) and (UDFToDouble(k1) > 20.0D)) (type: boolean) - Statistics: Num rows: 55 Data size: 19360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: k1 (type: string), k3 (type: string), k4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 55 Data size: 14575 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 14575 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: _col1 (type: string), _col2 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k1":"true","k2":"true","k3":"true","k4":"true"}} - bucket_count -1 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 11124 - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11624 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k1,k2,k3,k4 - columns.comments - columns.types string:string:string:string -#### A masked pattern was here #### - name default.dst_union22 - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22 - name: default.dst_union22 -#### A masked pattern was here #### - Partition - base file name: ds=1 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 1 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"k0":"true","k1":"true","k2":"true","k3":"true","k4":"true","k5":"true"}} - bucket_count -1 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - numFiles 1 - numRows 500 - partition_columns ds - partition_columns.types string - rawDataSize 16936 - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 17436 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns k0,k1,k2,k3,k4,k5 - columns.comments - columns.types string:string:string:string:string:string -#### A masked pattern was here #### - name default.dst_union22_delta - partition_columns ds - partition_columns.types string - serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dst_union22_delta - name: default.dst_union22_delta - Truncated Path -> Alias: - /dst_union22/ds=1 [$hdt$_0-subquery2:$hdt$_0:a] - /dst_union22_delta/ds=1 [$hdt$_0-subquery2:$hdt$_1:dst_union22_delta] - Needs Tagging: true - Reduce Operator Tree: - Join Operator - condition map: - Left Outer Join 0 to 1 - filter mappings: - 0 [1, 1] - filter predicates: - 0 {VALUE._col1} - 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col4, _col5 - Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 221 Data size: 49306 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') select * from ( diff --git ql/src/test/results/clientpositive/llap/union24.q.out ql/src/test/results/clientpositive/llap/union24.q.out new file mode 100644 index 0000000000..48f594cd1a --- /dev/null +++ ql/src/test/results/clientpositive/llap/union24.q.out @@ -0,0 +1,1599 @@ +PREHOOK: query: create table src2_n6 as select key, count(1) as count from src group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src2_n6 +POSTHOOK: query: create table src2_n6 as select key, count(1) as count from src group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src2_n6 +POSTHOOK: Lineage: src2_n6.count EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: src2_n6.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: create table src3_n2 as select * from src2_n6 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2_n6 +PREHOOK: Output: database:default +PREHOOK: Output: default@src3_n2 +POSTHOOK: query: create table src3_n2 as select * from src2_n6 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src3_n2 +POSTHOOK: Lineage: src3_n2.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src3_n2.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: create table src4_n0 as select * from src2_n6 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2_n6 +PREHOOK: Output: database:default +PREHOOK: Output: default@src4_n0 +POSTHOOK: query: create table src4_n0 as select * from src2_n6 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src4_n0 +POSTHOOK: Lineage: src4_n0.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src4_n0.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: create table src5_n3 as select * from src2_n6 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2_n6 +PREHOOK: Output: database:default +PREHOOK: Output: default@src5_n3 +POSTHOOK: query: create table src5_n3 as select * from src2_n6 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5_n3 +POSTHOOK: Lineage: src5_n3.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] +POSTHOOK: Lineage: src5_n3.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select key, count from src4_n0 where key < 10 + union all + select key, count(1) as count from src5_n3 where key < 10 group by key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select key, count from src4_n0 where key < 10 + union all + select key, count(1) as count from src5_n3 where key < 10 group by key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +OPTIMIZED SQL: SELECT `key`, `count` +FROM `default`.`src2_n6` +WHERE `key` < 10 +UNION ALL +SELECT `key`, `count` +FROM `default`.`src3_n2` +WHERE `key` < 10 +UNION ALL +SELECT `key`, `count` +FROM `default`.`src4_n0` +WHERE `key` < 10 +UNION ALL +SELECT `key`, COUNT(*) AS `$f1` +FROM `default`.`src5_n3` +WHERE `key` < 10 +GROUP BY `key` +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2_n6 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2_n6 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2_n6 + name: default.src2_n6 + Truncated Path -> Alias: + /src2_n6 [src2_n6] + Map 3 + Map Operator Tree: + TableScan + alias: src3_n2 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3_n2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3_n2 + name: default.src3_n2 + Truncated Path -> Alias: + /src3_n2 [src3_n2] + Map 4 + Map Operator Tree: + TableScan + alias: src4_n0 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4_n0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4_n0 + name: default.src4_n0 + Truncated Path -> Alias: + /src4_n0 [src4_n0] + Map 5 + Map Operator Tree: + TableScan + alias: src5_n3 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5048544 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5_n3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5_n3 + name: default.src5_n3 + Truncated Path -> Alias: + /src5_n3 [src5_n3] + Reducer 6 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select key, count from src4_n0 where key < 10 + union all + select key, count(1) as count from src5_n3 where key < 10 group by key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select key, count from src4_n0 where key < 10 + union all + select key, count(1) as count from src5_n3 where key < 10 group by key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +0 1 +0 3 +0 3 +0 3 +2 1 +2 1 +2 1 +2 1 +4 1 +4 1 +4 1 +4 1 +5 1 +5 3 +5 3 +5 3 +8 1 +8 1 +8 1 +8 1 +9 1 +9 1 +9 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +OPTIMIZED SQL: SELECT `key`, `count` +FROM `default`.`src2_n6` +WHERE `key` < 10 +UNION ALL +SELECT `key`, `count` +FROM `default`.`src3_n2` +WHERE `key` < 10 +UNION ALL +SELECT `t4`.`key`, `t6`.`count` +FROM (SELECT `key` +FROM `default`.`src4_n0` +WHERE `key` < 10) AS `t4` +INNER JOIN (SELECT `key`, `count` +FROM `default`.`src5_n3` +WHERE `key` < 10) AS `t6` ON `t4`.`key` = `t6`.`key` +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2_n6 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2_n6 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2_n6 + name: default.src2_n6 + Truncated Path -> Alias: + /src2_n6 [src2_n6] + Map 3 + Map Operator Tree: + TableScan + alias: src3_n2 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3_n2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3_n2 + name: default.src3_n2 + Truncated Path -> Alias: + /src3_n2 [src3_n2] + Map 4 + Map Operator Tree: + TableScan + alias: a + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4_n0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4_n0 + name: default.src4_n0 + Truncated Path -> Alias: + /src4_n0 [a] + Map 6 + Map Operator Tree: + TableScan + alias: b + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5_n3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5_n3 + name: default.src5_n3 + Truncated Path -> Alias: + /src5_n3 [b] + Reducer 5 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + Position of Big Table: 1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +0 3 +0 3 +0 3 +2 1 +2 1 +2 1 +4 1 +4 1 +4 1 +5 3 +5 3 +5 3 +8 1 +8 1 +8 1 +9 1 +9 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +OPTIMIZED SQL: SELECT `key`, `count` +FROM `default`.`src2_n6` +WHERE `key` < 10 +UNION ALL +SELECT `key`, `count` +FROM `default`.`src3_n2` +WHERE `key` < 10 +UNION ALL +SELECT `t4`.`key`, COUNT(*) AS `$f1` +FROM (SELECT `key` +FROM `default`.`src4_n0` +WHERE `key` < 10) AS `t4` +INNER JOIN (SELECT `key` +FROM `default`.`src5_n3` +WHERE `key` < 10) AS `t6` ON `t4`.`key` = `t6`.`key` +GROUP BY `t4`.`key` +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2_n6 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2_n6 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2_n6 + numFiles 2 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2_n6 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2_n6 + name: default.src2_n6 + Truncated Path -> Alias: + /src2_n6 [src2_n6] + Map 3 + Map Operator Tree: + TableScan + alias: src3_n2 + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3_n2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3_n2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3_n2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3_n2 + name: default.src3_n2 + Truncated Path -> Alias: + /src3_n2 [src3_n2] + Map 4 + Map Operator Tree: + TableScan + alias: a + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4_n0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4_n0 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4_n0 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4_n0 + name: default.src4_n0 + Truncated Path -> Alias: + /src4_n0 [a] + Map 7 + Map Operator Tree: + TableScan + alias: b + filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0D) (type: boolean) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5_n3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5_n3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5_n3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5_n3 + name: default.src5_n3 + Truncated Path -> Alias: + /src5_n3 [b] + Reducer 5 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Position of Big Table: 0 + Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.5048544 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Reducer 6 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2_n6 +PREHOOK: Input: default@src3_n2 +PREHOOK: Input: default@src4_n0 +PREHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2_n6 where key < 10 + union all + select key, count from src3_n2 where key < 10 + union all + select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2_n6 +POSTHOOK: Input: default@src3_n2 +POSTHOOK: Input: default@src4_n0 +POSTHOOK: Input: default@src5_n3 +#### A masked pattern was here #### +0 1 +0 3 +0 3 +2 1 +2 1 +2 1 +4 1 +4 1 +4 1 +5 1 +5 3 +5 3 +8 1 +8 1 +8 1 +9 1 +9 1 +9 1 diff --git ql/src/test/results/clientpositive/llap/union25.q.out ql/src/test/results/clientpositive/llap/union25.q.out new file mode 100644 index 0000000000..144582195d --- /dev/null +++ ql/src/test/results/clientpositive/llap/union25.q.out @@ -0,0 +1,271 @@ +PREHOOK: query: create table tmp_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: create table tmp_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION ALL + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION ALL + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tmp_srcpart +PREHOOK: Input: default@tmp_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_unionall +POSTHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION ALL + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION ALL + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tmp_srcpart +POSTHOOK: Input: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_unionall +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 6 (CONTAINS) + Map 8 <- Union 6 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: a + filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.684 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: b + filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.684 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string) + outputColumnNames: col1, col2, col3 + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll'), compute_stats(col3, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 316 Data size: 54984 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Union 2 + Vertex: Union 2 + Union 6 + Vertex: Union 6 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create Table + columns: counts bigint, key string, value string + name: default.tmp_unionall + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: counts, key, value + Column Types: bigint, string, string + Table: default.tmp_unionall + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + diff --git ql/src/test/results/clientpositive/union26.q.out ql/src/test/results/clientpositive/llap/union26.q.out similarity index 69% rename from ql/src/test/results/clientpositive/union26.q.out rename to ql/src/test/results/clientpositive/llap/union26.q.out index b0fe0c0373..a5b83201cb 100644 --- ql/src/test/results/clientpositive/union26.q.out +++ ql/src/test/results/clientpositive/llap/union26.q.out @@ -52,157 +52,163 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: ((ds = '2008-04-08') and (hr = '11') and key is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - TableScan - alias: b - filterExpr: ((ds = '2008-04-08') and (hr = '12') and key is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 1791 Data size: 318798 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 351000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 379000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 379000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - TableScan - alias: srcpart - filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 351000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 379000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1791 Data size: 318798 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 379000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1791 Data size: 318798 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 895 Data size: 166470 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union27.q.out ql/src/test/results/clientpositive/llap/union27.q.out new file mode 100644 index 0000000000..21549b9970 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union27.q.out @@ -0,0 +1,178 @@ +PREHOOK: query: create table jackson_sev_same_n0 as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_same_n0 +POSTHOOK: query: create table jackson_sev_same_n0 as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_same_n0 +POSTHOOK: Lineage: jackson_sev_same_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: jackson_sev_same_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table dim_pho_n0 as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dim_pho_n0 +POSTHOOK: query: create table dim_pho_n0 as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dim_pho_n0 +POSTHOOK: Lineage: dim_pho_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dim_pho_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table jackson_sev_add_n0 as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_add_n0 +POSTHOOK: query: create table jackson_sev_add_n0 as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_add_n0 +POSTHOOK: Lineage: jackson_sev_add_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: jackson_sev_add_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +PREHOOK: Input: default@dim_pho_n0 +PREHOOK: Input: default@jackson_sev_add_n0 +PREHOOK: Input: default@jackson_sev_same_n0 +#### A masked pattern was here #### +POSTHOOK: query: explain select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dim_pho_n0 +POSTHOOK: Input: default@jackson_sev_add_n0 +POSTHOOK: Input: default@jackson_sev_same_n0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 5 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: dim_pho_n0 + filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: jackson_sev_add_n0 + filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 97.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +PREHOOK: Input: default@dim_pho_n0 +PREHOOK: Input: default@jackson_sev_add_n0 +PREHOOK: Input: default@jackson_sev_same_n0 +#### A masked pattern was here #### +POSTHOOK: query: select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dim_pho_n0 +POSTHOOK: Input: default@jackson_sev_add_n0 +POSTHOOK: Input: default@jackson_sev_same_n0 +#### A masked pattern was here #### +97 val_97 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +97 val_97 diff --git ql/src/test/results/clientpositive/llap/union28.q.out ql/src/test/results/clientpositive/llap/union28.q.out new file mode 100644 index 0000000000..46505a54d8 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union28.q.out @@ -0,0 +1,306 @@ +PREHOOK: query: create table union_subq_union(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: create table union_subq_union(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union +PREHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.union_subq_union + +PREHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union +POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +2 val_2 +2 val_2 +2 val_2 +4 val_4 +4 val_4 +4 val_4 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +8 val_8 +8 val_8 +8 val_8 +9 val_9 diff --git ql/src/test/results/clientpositive/llap/union29.q.out ql/src/test/results/clientpositive/llap/union29.q.out new file mode 100644 index 0000000000..09bba31bbe --- /dev/null +++ ql/src/test/results/clientpositive/llap/union29.q.out @@ -0,0 +1,266 @@ +PREHOOK: query: create table union_subq_union_n1(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union_n1 +POSTHOOK: query: create table union_subq_union_n1(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union_n1 +PREHOOK: query: explain +insert overwrite table union_subq_union_n1 +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value from src + union all + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union_n1 +POSTHOOK: query: explain +insert overwrite table union_subq_union_n1 +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value from src + union all + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.union_subq_union_n1 + +PREHOOK: query: insert overwrite table union_subq_union_n1 +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value from src + union all + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union_n1 +POSTHOOK: query: insert overwrite table union_subq_union_n1 +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value from src + union all + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union_n1 +POSTHOOK: Lineage: union_subq_union_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union_n1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union_n1 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union_n1 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union_n1 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union_n1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +2 val_2 +2 val_2 +2 val_2 +4 val_4 +4 val_4 +4 val_4 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 diff --git ql/src/test/results/clientpositive/llap/union30.q.out ql/src/test/results/clientpositive/llap/union30.q.out new file mode 100644 index 0000000000..37cb6d6764 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union30.q.out @@ -0,0 +1,373 @@ +PREHOOK: query: create table union_subq_union_n2(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union_n2 +POSTHOOK: query: create table union_subq_union_n2(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union_n2 +PREHOOK: query: explain +insert overwrite table union_subq_union_n2 +select * from ( + +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a + +union all + +select key, value from src +) aa +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union_n2 +POSTHOOK: query: explain +insert overwrite table union_subq_union_n2 +select * from ( + +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a + +union all + +select key, value from src +) aa +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union_n2 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n2 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n2 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n2 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n2 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union_n2 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.union_subq_union_n2 + +PREHOOK: query: insert overwrite table union_subq_union_n2 +select * from ( + +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a + +union all + +select key, value from src +) aa +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union_n2 +POSTHOOK: query: insert overwrite table union_subq_union_n2 +select * from ( + +select * from ( + select key, value from src + union all + select key, value from + ( + select key, value, count(1) from src group by key, value + union all + select key, value, count(1) from src group by key, value + ) subq +) a + +union all + +select key, value from src +) aa +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union_n2 +POSTHOOK: Lineage: union_subq_union_n2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union_n2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union_n2 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union_n2 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union_n2 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union_n2 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +2 val_2 +2 val_2 +2 val_2 +2 val_2 +4 val_4 +4 val_4 +4 val_4 +4 val_4 +5 val_5 +5 val_5 +5 val_5 +5 val_5 diff --git ql/src/test/results/clientpositive/llap/union31.q.out ql/src/test/results/clientpositive/llap/union31.q.out new file mode 100644 index 0000000000..48df6c38a7 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union31.q.out @@ -0,0 +1,1134 @@ +PREHOOK: query: drop table t1_n28 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t1_n28 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table t2_n19 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t2_n19 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1_n28 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n28 +POSTHOOK: query: create table t1_n28 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n28 +POSTHOOK: Lineage: t1_n28.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1_n28.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table t2_n19 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2_n19 +POSTHOOK: query: create table t2_n19 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2_n19 +POSTHOOK: Lineage: t2_n19.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t2_n19.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table t3_n6(key string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3_n6 +POSTHOOK: query: create table t3_n6(key string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3_n6 +PREHOOK: query: create table t4_n2(value string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t4_n2 +POSTHOOK: query: create table t4_n2(value string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4_n2 +PREHOOK: query: explain +from +(select * from t1_n28 + union all + select * from t2_n19 +) x +insert overwrite table t3_n6 + select key, count(1) group by key +insert overwrite table t4_n2 + select value, count(1) group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t3_n6 +PREHOOK: Output: default@t4_n2 +POSTHOOK: query: explain +from +(select * from t1_n28 + union all + select * from t2_n19 +) x +insert overwrite table t3_n6 + select key, count(1) group by key +insert overwrite table t4_n2 + select value, count(1) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t3_n6 +POSTHOOK: Output: default@t4_n2 +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 7 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Union 2 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n28 + Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.7 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col1 (type: string) + minReductionHashAggr: 0.7 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: t2_n19 + Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + minReductionHashAggr: 0.7 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col1 (type: string) + minReductionHashAggr: 0.7 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3_n6 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: key, cnt + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8333333 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4_n2 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: value, cnt + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8333333 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3_n6 + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, cnt + Column Types: string, int + Table: default.t3_n6 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4_n2 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, cnt + Column Types: string, int + Table: default.t4_n2 + +PREHOOK: query: from +(select * from t1_n28 + union all + select * from t2_n19 +) x +insert overwrite table t3_n6 + select key, count(1) group by key +insert overwrite table t4_n2 + select value, count(1) group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t3_n6 +PREHOOK: Output: default@t4_n2 +POSTHOOK: query: from +(select * from t1_n28 + union all + select * from t2_n19 +) x +insert overwrite table t3_n6 + select key, count(1) group by key +insert overwrite table t4_n2 + select value, count(1) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t3_n6 +POSTHOOK: Output: default@t4_n2 +POSTHOOK: Lineage: t3_n6.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +POSTHOOK: Lineage: t3_n6.key EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t4_n2.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +POSTHOOK: Lineage: t4_n2.value EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:value, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from t3_n6 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3_n6 +#### A masked pattern was here #### +POSTHOOK: query: select * from t3_n6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3_n6 +#### A masked pattern was here #### +0 6 +2 2 +4 2 +5 6 +8 2 +9 2 +PREHOOK: query: select * from t4_n2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4_n2 +#### A masked pattern was here #### +POSTHOOK: query: select * from t4_n2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4_n2 +#### A masked pattern was here #### +val_0 6 +val_2 2 +val_4 2 +val_5 6 +val_8 2 +val_9 2 +PREHOOK: query: create table t5_n0(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t5_n0 +POSTHOOK: query: create table t5_n0(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t5_n0 +PREHOOK: query: create table t6_n0(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t6_n0 +POSTHOOK: query: create table t6_n0(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t6_n0 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, count(1) as cnt from t2_n19 group by key +) x +insert overwrite table t5_n0 + select c1, sum(cnt) group by c1 +insert overwrite table t6_n0 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t5_n0 +PREHOOK: Output: default@t6_n0 +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, count(1) as cnt from t2_n19 group by key +) x +insert overwrite table t5_n0 + select c1, sum(cnt) group by c1 +insert overwrite table t6_n0 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t5_n0 +POSTHOOK: Output: default@t6_n0 +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n28 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: t2_n19 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Forward + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5_n0 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: c1, cnt + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6_n0 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: c1, cnt + Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5_n0 + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, cnt + Column Types: string, int + Table: default.t5_n0 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6_n0 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, cnt + Column Types: string, int + Table: default.t6_n0 + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, count(1) as cnt from t2_n19 group by key +) x +insert overwrite table t5_n0 + select c1, sum(cnt) group by c1 +insert overwrite table t6_n0 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t5_n0 +PREHOOK: Output: default@t6_n0 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, count(1) as cnt from t2_n19 group by key +) x +insert overwrite table t5_n0 + select c1, sum(cnt) group by c1 +insert overwrite table t6_n0 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t5_n0 +POSTHOOK: Output: default@t6_n0 +POSTHOOK: Lineage: t5_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t5_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +POSTHOOK: Lineage: t6_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t6_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +PREHOOK: query: select * from t5_n0 +PREHOOK: type: QUERY +PREHOOK: Input: default@t5_n0 +#### A masked pattern was here #### +POSTHOOK: query: select * from t5_n0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t5_n0 +#### A masked pattern was here #### +0 6 +2 2 +4 2 +5 6 +8 2 +9 2 +PREHOOK: query: select * from t6_n0 +PREHOOK: type: QUERY +PREHOOK: Input: default@t6_n0 +#### A masked pattern was here #### +POSTHOOK: query: select * from t6_n0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t6_n0 +#### A masked pattern was here #### +0 6 +2 2 +4 2 +5 6 +8 2 +9 2 +PREHOOK: query: drop table t1_n28 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_n28 +PREHOOK: Output: default@t1_n28 +POSTHOOK: query: drop table t1_n28 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Output: default@t1_n28 +PREHOOK: query: drop table t2_n19 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t2_n19 +POSTHOOK: query: drop table t2_n19 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t2_n19 +PREHOOK: query: create table t1_n28 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n28 +POSTHOOK: query: create table t1_n28 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n28 +POSTHOOK: Lineage: t1_n28.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1_n28.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table t2_n19 as select key, count(1) as cnt from src where key < 10 group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2_n19 +POSTHOOK: query: create table t2_n19 as select key, count(1) as cnt from src where key < 10 group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2_n19 +POSTHOOK: Lineage: t2_n19.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: t2_n19.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: create table t7_n1(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t7_n1 +POSTHOOK: query: create table t7_n1(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t7_n1 +PREHOOK: query: create table t8_n0(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8_n0 +POSTHOOK: query: create table t8_n0(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8_n0 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, cnt from t2_n19 +) x +insert overwrite table t7_n1 + select c1, count(1) group by c1 +insert overwrite table t8_n0 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t7_n1 +PREHOOK: Output: default@t8_n0 +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, cnt from t2_n19 +) x +insert overwrite table t7_n1 + select c1, count(1) group by c1 +insert overwrite table t8_n0 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t7_n1 +POSTHOOK: Output: default@t8_n0 +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n28 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: t2_n19 + Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Forward + Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: c1, cnt + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8333333 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8_n0 + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: c1, cnt + Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') + minReductionHashAggr: 0.8333333 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7_n1 + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, cnt + Column Types: string, int + Table: default.t7_n1 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8_n0 + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: c1, cnt + Column Types: string, int + Table: default.t8_n0 + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, cnt from t2_n19 +) x +insert overwrite table t7_n1 + select c1, count(1) group by c1 +insert overwrite table t8_n0 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n28 +PREHOOK: Input: default@t2_n19 +PREHOOK: Output: default@t7_n1 +PREHOOK: Output: default@t8_n0 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1_n28 group by key + union all + select key as c1, cnt from t2_n19 +) x +insert overwrite table t7_n1 + select c1, count(1) group by c1 +insert overwrite table t8_n0 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n28 +POSTHOOK: Input: default@t2_n19 +POSTHOOK: Output: default@t7_n1 +POSTHOOK: Output: default@t8_n0 +POSTHOOK: Lineage: t7_n1.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t7_n1.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +POSTHOOK: Lineage: t8_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t8_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] +PREHOOK: query: select * from t7_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t7_n1 +#### A masked pattern was here #### +POSTHOOK: query: select * from t7_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t7_n1 +#### A masked pattern was here #### +0 2 +2 2 +4 2 +5 2 +8 2 +9 2 +PREHOOK: query: select * from t8_n0 +PREHOOK: type: QUERY +PREHOOK: Input: default@t8_n0 +#### A masked pattern was here #### +POSTHOOK: query: select * from t8_n0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8_n0 +#### A masked pattern was here #### +0 2 +2 2 +4 2 +5 2 +8 2 +9 2 diff --git ql/src/test/results/clientpositive/llap/union32.q.out ql/src/test/results/clientpositive/llap/union32.q.out new file mode 100644 index 0000000000..46a072163e --- /dev/null +++ ql/src/test/results/clientpositive/llap/union32.q.out @@ -0,0 +1,822 @@ +PREHOOK: query: CREATE TABLE t1_n70 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n70 +POSTHOOK: query: CREATE TABLE t1_n70 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n70 +POSTHOOK: Lineage: t1_n70.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1_n70.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE TABLE t2_n43 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2_n43 +POSTHOOK: query: CREATE TABLE t2_n43 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2_n43 +POSTHOOK: Lineage: t2_n43.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t2_n43.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 +UNION ALL +SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 +UNION ALL +SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n70 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t2_n43 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(UDFToLong(key)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 +UNION ALL +SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 +UNION ALL +SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +2.0 +2.0 +4.0 +4.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +8.0 +8.0 +9.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToLong(key) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: t2_n43 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(_col1) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +2.0 +2.0 +4.0 +4.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +8.0 +8.0 +9.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2_n43 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToLong(key) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(_col1) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +0.0 +2.0 +2.0 +4.0 +4.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +5.0 +8.0 +8.0 +9.0 +9.0 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToLong(key) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), CAST( key AS CHAR(20)) (type: char(20)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(20)) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: t2_n43 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(key) (type: double), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 16 Data size: 1792 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(_col1) (type: double), CAST( _col3 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3072 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key +UNION ALL +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +2.0 2 +2.0 2 +4.0 4 +4.0 4 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +8.0 8 +8.0 8 +9.0 9 +9.0 9 +PREHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2_n43 + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(key) (type: double), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToLong(key) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), CAST( key AS CHAR(20)) (type: char(20)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(20)) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 16 Data size: 1792 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToDouble(_col1) (type: double), CAST( _col3 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3072 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n70 +PREHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 +UNION ALL +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n70 +POSTHOOK: Input: default@t2_n43 +#### A masked pattern was here #### +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +0.0 0 +2.0 2 +2.0 2 +4.0 4 +4.0 4 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +5.0 5 +8.0 8 +8.0 8 +9.0 9 +9.0 9 diff --git ql/src/test/results/clientpositive/llap/union33.q.out ql/src/test/results/clientpositive/llap/union33.q.out new file mode 100644 index 0000000000..8fd7573715 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union33.q.out @@ -0,0 +1,460 @@ +PREHOOK: query: CREATE TABLE test_src_n1 (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_src_n1 +POSTHOOK: query: CREATE TABLE test_src_n1 (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_src_n1 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION ALL + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src_n1 +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION ALL + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 0.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 0.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: rand() (type: double) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: partials + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.test_src_n1 + +PREHOOK: query: INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION ALL + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src_n1 +POSTHOOK: query: INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION ALL + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src_n1 +POSTHOOK: Lineage: test_src_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src_n1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] +PREHOOK: query: SELECT COUNT(*) FROM test_src_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src_n1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src_n1 +#### A masked pattern was here #### +312 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION ALL + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src_n1 +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION ALL + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 6 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: rand() (type: double) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 0.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 0.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: partials + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src_n1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.test_src_n1 + +PREHOOK: query: INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION ALL + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src_n1 +POSTHOOK: query: INSERT OVERWRITE TABLE test_src_n1 +SELECT key, value FROM ( + SELECT key, cast(COUNT(*) as string) AS value FROM src + GROUP BY key +UNION ALL + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src_n1 +POSTHOOK: Lineage: test_src_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src_n1.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM test_src_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src_n1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src_n1 +#### A masked pattern was here #### +312 diff --git ql/src/test/results/clientpositive/llap/union34.q.out ql/src/test/results/clientpositive/llap/union34.q.out new file mode 100644 index 0000000000..3bc9da3048 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union34.q.out @@ -0,0 +1,446 @@ +PREHOOK: query: create table src10_1_n0 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_1_n0 +POSTHOOK: query: create table src10_1_n0 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_1_n0 +PREHOOK: query: create table src10_2_n0 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_2_n0 +POSTHOOK: query: create table src10_2_n0 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_2_n0 +PREHOOK: query: create table src10_3_n0 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_3_n0 +POSTHOOK: query: create table src10_3_n0 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_3_n0 +PREHOOK: query: create table src10_4_n0 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_4_n0 +POSTHOOK: query: create table src10_4_n0 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_4_n0 +PREHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1_n0 select * +insert overwrite table src10_2_n0 select * +insert overwrite table src10_3_n0 select * +insert overwrite table src10_4_n0 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src10_1_n0 +PREHOOK: Output: default@src10_2_n0 +PREHOOK: Output: default@src10_3_n0 +PREHOOK: Output: default@src10_4_n0 +POSTHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1_n0 select * +insert overwrite table src10_2_n0 select * +insert overwrite table src10_3_n0 select * +insert overwrite table src10_4_n0 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src10_1_n0 +POSTHOOK: Output: default@src10_2_n0 +POSTHOOK: Output: default@src10_3_n0 +POSTHOOK: Output: default@src10_4_n0 +POSTHOOK: Lineage: src10_1_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_1_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1_n0 +PREHOOK: Input: default@src10_2_n0 +PREHOOK: Input: default@src10_3_n0 +PREHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +POSTHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1_n0 +POSTHOOK: Input: default@src10_2_n0 +POSTHOOK: Input: default@src10_3_n0 +POSTHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src10_1_n0 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 3 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: src10_2_n0 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src10_3_n0 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src10_4_n0 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1_n0 +PREHOOK: Input: default@src10_2_n0 +PREHOOK: Input: default@src10_3_n0 +PREHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1_n0 +POSTHOOK: Input: default@src10_2_n0 +POSTHOOK: Input: default@src10_3_n0 +POSTHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +165 val_165 +165 val_165 +165 val_165 +238 val_238 +238 val_238 +238 val_238 +255 val_255 +255 val_255 +255 val_255 +27 val_27 +27 val_27 +27 val_27 +278 val_278 +278 val_278 +278 val_278 +311 val_311 +311 val_311 +311 val_311 +409 val_409 +409 val_409 +409 val_409 +484 val_484 +484 val_484 +484 val_484 +86 val_86 +86 val_86 +86 val_86 +98 val_98 +98 val_98 +98 val_98 +PREHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1_n0 +PREHOOK: Input: default@src10_2_n0 +PREHOOK: Input: default@src10_3_n0 +PREHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +POSTHOOK: query: explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1_n0 +POSTHOOK: Input: default@src10_2_n0 +POSTHOOK: Input: default@src10_3_n0 +POSTHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src10_1_n0 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src10_2_n0 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src10_3_n0 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: src10_4_n0 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1_n0 +PREHOOK: Input: default@src10_2_n0 +PREHOOK: Input: default@src10_3_n0 +PREHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) + UNION ALL + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 +) alias1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1_n0 +POSTHOOK: Input: default@src10_2_n0 +POSTHOOK: Input: default@src10_3_n0 +POSTHOOK: Input: default@src10_4_n0 +#### A masked pattern was here #### +165 val_165 +165 val_165 +165 val_165 +238 val_238 +238 val_238 +238 val_238 +255 val_255 +255 val_255 +255 val_255 +27 val_27 +27 val_27 +27 val_27 +278 val_278 +278 val_278 +278 val_278 +311 val_311 +311 val_311 +311 val_311 +409 val_409 +409 val_409 +409 val_409 +484 val_484 +484 val_484 +484 val_484 +86 val_86 +86 val_86 +86 val_86 +98 val_98 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/union35.q.out ql/src/test/results/clientpositive/llap/union35.q.out similarity index 100% rename from ql/src/test/results/clientpositive/union35.q.out rename to ql/src/test/results/clientpositive/llap/union35.q.out index 2bf03274e2..105dee6dc7 100644 --- ql/src/test/results/clientpositive/union35.q.out +++ ql/src/test/results/clientpositive/llap/union35.q.out @@ -15,15 +15,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 1 foo -2 bar 1 foo -2 bar 1 foo -2 bar 1 foo -2 bar 1 foo 2 bar +2 bar +2 bar +2 bar +2 bar PREHOOK: query: select * from ( select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f union all @@ -41,12 +41,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 1 foo -2 bar 1 foo -2 bar 1 foo -2 bar 1 foo -2 bar 1 foo 2 bar +2 bar +2 bar +2 bar +2 bar diff --git ql/src/test/results/clientpositive/llap/union37.q.out ql/src/test/results/clientpositive/llap/union37.q.out new file mode 100644 index 0000000000..8a4db395cb --- /dev/null +++ ql/src/test/results/clientpositive/llap/union37.q.out @@ -0,0 +1,590 @@ +PREHOOK: query: create table l_test1 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@l_test1 +POSTHOOK: query: create table l_test1 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@l_test1 +PREHOOK: query: insert into l_test1 values (1, "table_1", "2016-08-11") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@l_test1 +POSTHOOK: query: insert into l_test1 values (1, "table_1", "2016-08-11") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@l_test1 +POSTHOOK: Lineage: l_test1.id SCRIPT [] +POSTHOOK: Lineage: l_test1.trans_date SCRIPT [] +POSTHOOK: Lineage: l_test1.val SCRIPT [] +PREHOOK: query: create table l_test2 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@l_test2 +POSTHOOK: query: create table l_test2 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@l_test2 +PREHOOK: query: insert into l_test2 values (2, "table_2", "2016-08-11") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@l_test2 +POSTHOOK: query: insert into l_test2 values (2, "table_2", "2016-08-11") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@l_test2 +POSTHOOK: Lineage: l_test2.id SCRIPT [] +POSTHOOK: Lineage: l_test2.trans_date SCRIPT [] +POSTHOOK: Lineage: l_test2.val SCRIPT [] +PREHOOK: query: explain +select + id, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: explain +select + id, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: l_test1 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: l_test2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), val (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select + id, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: select + id, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +1 table_1 2016-08-11 +2 table_2 2016-08-11 +PREHOOK: query: explain +select + id, + 999, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: explain +select + id, + 999, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: l_test1 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: l_test2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), val (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select + id, + 999, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: select + id, + 999, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +1 999 table_1 2016-08-11 +2 999 table_2 2016-08-11 +PREHOOK: query: explain +select + id, + 999, + 666, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + 666, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: explain +select + id, + 999, + 666, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + 666, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: l_test1 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), 666 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: l_test2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), val (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), 666 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select + id, + 999, + 666, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + 666, + val, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: select + id, + 999, + 666, + 'table_1' , + trans_date +from l_test1 +union all +select + id, + 999, + 666, + val, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +1 999 666 table_1 2016-08-11 +2 999 666 table_2 2016-08-11 +PREHOOK: query: explain +select + id, + 999, + 'table_1' , + trans_date, + '2016-11-11' +from l_test1 +union all +select + id, + 999, + val, + trans_date, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: explain +select + id, + 999, + 'table_1' , + trans_date, + '2016-11-11' +from l_test1 +union all +select + id, + 999, + val, + trans_date, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: l_test1 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string), '2016-11-11' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 287 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: l_test2 + Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint), val (type: string), trans_date (type: string), trans_date (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 287 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select + id, + 999, + 'table_1' , + trans_date, + '2016-11-11' +from l_test1 +union all +select + id, + 999, + val, + trans_date, + trans_date +from l_test2 +PREHOOK: type: QUERY +PREHOOK: Input: default@l_test1 +PREHOOK: Input: default@l_test2 +#### A masked pattern was here #### +POSTHOOK: query: select + id, + 999, + 'table_1' , + trans_date, + '2016-11-11' +from l_test1 +union all +select + id, + 999, + val, + trans_date, + trans_date +from l_test2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@l_test1 +POSTHOOK: Input: default@l_test2 +#### A masked pattern was here #### +1 999 table_1 2016-08-11 2016-11-11 +2 999 table_2 2016-08-11 2016-08-11 diff --git ql/src/test/results/clientpositive/llap/union_lateralview.q.out ql/src/test/results/clientpositive/llap/union_lateralview.q.out new file mode 100644 index 0000000000..d9d4611d78 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_lateralview.q.out @@ -0,0 +1,341 @@ +PREHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_union_lateral_view +POSTHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_union_lateral_view +PREHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@test_union_lateral_view +POSTHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@test_union_lateral_view +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Map 6 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), array(1,2,3) (type: array) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 117000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 290000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Select Operator + expressions: _col2 (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ((ds = '2008-04-08') and (hr = '12')) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), array(1,2,3) (type: array) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 117000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Forward + Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 290000 Basic stats: COMPLETE Column stats: COMPLETE + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Select Operator + expressions: _col2 (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 3104 Data size: 468967 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col3) (type: int), _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_union_lateral_view + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + outputColumnNames: key, arr_ele, value + Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(arr_ele, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_union_lateral_view + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, arr_ele, value + Column Types: int, int, string + Table: default.test_union_lateral_view + +PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@test_union_lateral_view +POSTHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@test_union_lateral_view +POSTHOOK: Lineage: test_union_lateral_view.arr_ele SCRIPT [] +POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_union_lateral_view +#### A masked pattern was here #### +POSTHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_union_lateral_view +#### A masked pattern was here #### +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 2 val_0 +0 2 val_0 diff --git ql/src/test/results/clientpositive/llap/union_offcbo.q.out ql/src/test/results/clientpositive/llap/union_offcbo.q.out new file mode 100644 index 0000000000..0c12d66bc8 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_offcbo.q.out @@ -0,0 +1,1858 @@ +PREHOOK: query: DROP TABLE IF EXISTS ttest1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS ttest1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS ttest2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS ttest2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE ttest1 ( + `id1` bigint COMMENT 'from deserializer', + `ts1` string COMMENT 'from deserializer', + `dt1` string COMMENT 'from deserializer', + `dt2` string COMMENT 'from deserializer', + `ac1` string COMMENT 'from deserializer', + `kd1` string COMMENT 'from deserializer', + `sts` string COMMENT 'from deserializer', + `at1` bigint COMMENT 'from deserializer') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ttest1 +POSTHOOK: query: CREATE TABLE ttest1 ( + `id1` bigint COMMENT 'from deserializer', + `ts1` string COMMENT 'from deserializer', + `dt1` string COMMENT 'from deserializer', + `dt2` string COMMENT 'from deserializer', + `ac1` string COMMENT 'from deserializer', + `kd1` string COMMENT 'from deserializer', + `sts` string COMMENT 'from deserializer', + `at1` bigint COMMENT 'from deserializer') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ttest1 +PREHOOK: query: CREATE TABLE ttest2 ( + `id1` bigint, + `ts1` string, + `dt1` string, + `dt2` string, + `ac1` string, + `kd1` string, + `sts` string, + `at1` bigint, + `khash` string, + `rhash` string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ttest2 +POSTHOOK: query: CREATE TABLE ttest2 ( + `id1` bigint, + `ts1` string, + `dt1` string, + `dt2` string, + `ac1` string, + `kd1` string, + `sts` string, + `at1` bigint, + `khash` string, + `rhash` string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ttest2 +PREHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +PREHOOK: type: QUERY +PREHOOK: Input: default@ttest1 +PREHOOK: Input: default@ttest2 +#### A masked pattern was here #### +POSTHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ttest1 +POSTHOOK: Input: default@ttest2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col8, _col9 + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + value expressions: _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ts1 (type: string), khash (type: string), rhash (type: string) + outputColumnNames: _col1, _col8, _col9 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and ((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END <> 'RET') and ((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +PREHOOK: type: QUERY +PREHOOK: Input: default@ttest1 +PREHOOK: Input: default@ttest2 +#### A masked pattern was here #### +POSTHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ttest1 +POSTHOOK: Input: default@ttest2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col5 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col5 (type: string) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint), _col6 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col3 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ts1 (type: string), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 is null and (_col3 >= '2016-02-05')) or (_col0 is null and (_col3 <= '2016-02-05')) is not true) and ((COALESCE(_col0,-1) <> COALESCE(_col7,-1)) or (COALESCE(_col1,-1) <> COALESCE(_col8,-1)))) (type: boolean) + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((COALESCE(_col3,-1) <> COALESCE(_col6,-1)) or (COALESCE(_col4,-1) <> COALESCE(_col7,-1))) (type: boolean) + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +PREHOOK: type: QUERY +PREHOOK: Input: default@ttest1 +PREHOOK: Input: default@ttest2 +#### A masked pattern was here #### +POSTHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ttest1 +POSTHOOK: Input: default@ttest2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col8, _col9 + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + value expressions: _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: ttest1 + filterExpr: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: ttest2 + filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ts1 (type: string), khash (type: string), rhash (type: string) + outputColumnNames: _col1, _col8, _col9 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col7 <> 'RET') (type: boolean) + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col7 <> 'RET') (type: boolean) + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +PREHOOK: type: QUERY +PREHOOK: Input: default@ttest1 +PREHOOK: Input: default@ttest2 +#### A masked pattern was here #### +POSTHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ttest1 +POSTHOOK: Input: default@ttest2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ttest1 + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col8, _col9 + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE + value expressions: _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: ttest2 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: ttest1 + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: ttest2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ts1 (type: string), khash (type: string), rhash (type: string) + outputColumnNames: _col1, _col8, _col9 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col8 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col8 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col9 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col7 <> 'RET') (type: boolean) + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col8 (type: string) + 1 _col8 (type: string) + outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col7 <> 'RET') (type: boolean) + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +PREHOOK: type: QUERY +PREHOOK: Input: default@ttest1 +PREHOOK: Input: default@ttest2 +#### A masked pattern was here #### +POSTHOOK: query: explain SELECT + A2.id1, A2.sts,A2.at1, + CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,A2.FLAG + FROM ( + SELECT + A2.id1, A2.sts,A2.at1 + ,A2.dt1 + ,A2.dt2 + ,A2.khash + ,A2.rhash + ,CASE + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 >= '2016-02-05' + THEN 'DEL' + + WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.dt1 <= '2016-02-05' + THEN 'RET' + + WHEN + ( + A2.khash = A1.khash + AND A2.rhash <> A1.rhash + ) + THEN 'A_INS' + + ELSE 'NA' + END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 AND dt2 + ) A2 + ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A2.khash IS NOT NULL + + UNION ALL + + SELECT A1.id1, A1.sts,A1.at1 + ,A1.dt1 + , '2099-12-31' dt2 + ,A1.khash + ,A1.rhash + ,CASE WHEN A2.khash IS NOT NULL + AND A1.khash IS NULL + AND A2.ts1 <= A1.ts1 + THEN 'DEL' + + WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) + OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG + FROM ( + SELECT * + ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash + ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash + FROM ttest1 + WHERE ts1 = '2015-11-20' + ) A1 + FULL OUTER JOIN ( + SELECT * + FROM ttest2 + WHERE '2015-11-20' BETWEEN dt1 + AND dt2 + ) A2 ON A1.khash = A2.khash + WHERE NOT ( + NVL(A1.khash, - 1) = NVL(A2.khash, - 1) + AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) + ) + AND A1.khash IS NOT NULL + ) A2 + where a2.flag <> 'RET' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ttest1 +POSTHOOK: Input: default@ttest2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ttest1 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: ttest2 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col5 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col5 (type: string) + Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint), _col6 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: ttest1 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (ts1 = '2015-11-20') (type: boolean) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col3 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: ttest2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ts1 (type: string), khash (type: string), rhash (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 is null and (_col3 >= '2016-02-05')) or (_col0 is null and (_col3 <= '2016-02-05')) is not true) and ((COALESCE(_col0,-1) <> COALESCE(_col7,-1)) or (COALESCE(_col1,-1) <> COALESCE(_col8,-1)))) (type: boolean) + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((COALESCE(_col3,-1) <> COALESCE(_col6,-1)) or (COALESCE(_col4,-1) <> COALESCE(_col7,-1))) (type: boolean) + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP TABLE ttest1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ttest1 +PREHOOK: Output: default@ttest1 +POSTHOOK: query: DROP TABLE ttest1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ttest1 +POSTHOOK: Output: default@ttest1 +PREHOOK: query: DROP TABLE ttest2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ttest2 +PREHOOK: Output: default@ttest2 +POSTHOOK: query: DROP TABLE ttest2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ttest2 +POSTHOOK: Output: default@ttest2 diff --git ql/src/test/results/clientpositive/union_paren.q.out ql/src/test/results/clientpositive/llap/union_paren.q.out similarity index 78% rename from ql/src/test/results/clientpositive/union_paren.q.out rename to ql/src/test/results/clientpositive/llap/union_paren.q.out index 7843045faf..3d7655e3e6 100644 --- ql/src/test/results/clientpositive/union_paren.q.out +++ ql/src/test/results/clientpositive/llap/union_paren.q.out @@ -12,40 +12,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union_pos_alias.q.out ql/src/test/results/clientpositive/llap/union_pos_alias.q.out new file mode 100644 index 0000000000..c7029c6628 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_pos_alias.q.out @@ -0,0 +1,2125 @@ +PREHOOK: query: explain +select 'tst1' as key, count(1) as value from src s1 +UNION ALL +select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 +order by 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain +select 'tst1' as key, count(1) as value from src s1 +UNION ALL +select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 +order by 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'tst1' as key, count(1) as value from src s1 +UNION ALL +select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 +order by 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select 'tst1' as key, count(1) as value from src s1 +UNION ALL +select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 +order by 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 500 +tst2 500 +tst3 500 +PREHOOK: query: drop table src_10 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_10 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_10 as select * from src order by key, value limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_10 +POSTHOOK: query: create table src_10 as select * from src order by key, value limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_10 +POSTHOOK: Lineage: src_10.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_10.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +select key as value, value as key from src_10 +UNION ALL +select 'test', value from src_10 s3 +order by 2, 1 desc +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +#### A masked pattern was here #### +POSTHOOK: query: explain +select key as value, value as key from src_10 +UNION ALL +select 'test', value from src_10 s3 +order by 2, 1 desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + null sort order: zz + sort order: +- + Statistics: Num rows: 20 Data size: 3540 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 10 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'test' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + null sort order: zz + sort order: +- + Statistics: Num rows: 20 Data size: 3540 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key as value, value as key from src_10 +UNION ALL +select 'test', value from src_10 s3 +order by 2, 1 desc +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +#### A masked pattern was here #### +POSTHOOK: query: select key as value, value as key from src_10 +UNION ALL +select 'test', value from src_10 s3 +order by 2, 1 desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +test val_0 +test val_0 +test val_0 +test val_10 +test val_100 +test val_100 +test val_103 +test val_103 +test val_104 +test val_104 +PREHOOK: query: drop table src_10 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_10 +POSTHOOK: query: drop table src_10 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_10 +PREHOOK: query: drop view v_n8 +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v_n8 +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v_n8 as select key as k from src intersect all select key as k1 from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v_n8 +POSTHOOK: query: create view v_n8 as select key as k from src intersect all select key as k1 from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v_n8 +POSTHOOK: Lineage: v_n8.k SCRIPT [(src)src.null, (src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: desc formatted v_n8 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v_n8 +POSTHOOK: query: desc formatted v_n8 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v_n8 +# col_name data_type comment +k string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select key as k from src intersect all select key as k1 from src +Expanded Query: select `src`.`key` as `k` from `default`.`src` intersect all select `src`.`key` as `k1` from `default`.`src` +PREHOOK: query: create table masking_test_n9 as select cast(key as int) as key, value from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_n9 +POSTHOOK: query: create table masking_test_n9 as select cast(key as int) as key, value from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_n9 +POSTHOOK: Lineage: masking_test_n9.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: masking_test_n9.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +select * from masking_test_n9 union all select * from masking_test_n9 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +POSTHOOK: query: explain +select * from masking_test_n9 union all select * from masking_test_n9 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: masking_test_n9 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: masking_test_n9 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test_n9 union all select * from masking_test_n9 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test_n9 union all select * from masking_test_n9 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +10 val_10 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +105 val_105 +105 val_105 +11 val_11 +11 val_11 +111 val_111 +111 val_111 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +114 val_114 +114 val_114 +116 val_116 +116 val_116 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +126 val_126 +126 val_126 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +131 val_131 +131 val_131 +133 val_133 +133 val_133 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +136 val_136 +136 val_136 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +143 val_143 +145 val_145 +145 val_145 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +149 val_149 +149 val_149 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +150 val_150 +150 val_150 +152 val_152 +152 val_152 +152 val_152 +152 val_152 +153 val_153 +153 val_153 +155 val_155 +155 val_155 +156 val_156 +156 val_156 +157 val_157 +157 val_157 +158 val_158 +158 val_158 +160 val_160 +160 val_160 +162 val_162 +162 val_162 +163 val_163 +163 val_163 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +166 val_166 +166 val_166 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +17 val_17 +17 val_17 +170 val_170 +170 val_170 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +176 val_176 +176 val_176 +177 val_177 +177 val_177 +178 val_178 +178 val_178 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +180 val_180 +180 val_180 +181 val_181 +181 val_181 +183 val_183 +183 val_183 +186 val_186 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +189 val_189 +19 val_19 +19 val_19 +190 val_190 +190 val_190 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +192 val_192 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +194 val_194 +195 val_195 +195 val_195 +195 val_195 +195 val_195 +196 val_196 +196 val_196 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +2 val_2 +2 val_2 +20 val_20 +20 val_20 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +201 val_201 +201 val_201 +202 val_202 +202 val_202 +203 val_203 +203 val_203 +203 val_203 +203 val_203 +205 val_205 +205 val_205 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +207 val_207 +207 val_207 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +209 val_209 +209 val_209 +209 val_209 +209 val_209 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +214 val_214 +214 val_214 +216 val_216 +216 val_216 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +218 val_218 +218 val_218 +219 val_219 +219 val_219 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +222 val_222 +222 val_222 +223 val_223 +223 val_223 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +226 val_226 +226 val_226 +228 val_228 +228 val_228 +229 val_229 +229 val_229 +229 val_229 +229 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +235 val_235 +235 val_235 +237 val_237 +237 val_237 +237 val_237 +237 val_237 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +241 val_241 +241 val_241 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +244 val_244 +244 val_244 +247 val_247 +247 val_247 +248 val_248 +248 val_248 +249 val_249 +249 val_249 +252 val_252 +252 val_252 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +256 val_256 +256 val_256 +256 val_256 +256 val_256 +257 val_257 +257 val_257 +258 val_258 +258 val_258 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +260 val_260 +260 val_260 +262 val_262 +262 val_262 +263 val_263 +263 val_263 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +266 val_266 +266 val_266 +27 val_27 +27 val_27 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +274 val_274 +275 val_275 +275 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +28 val_28 +28 val_28 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +283 val_283 +283 val_283 +284 val_284 +284 val_284 +285 val_285 +285 val_285 +286 val_286 +286 val_286 +287 val_287 +287 val_287 +288 val_288 +288 val_288 +288 val_288 +288 val_288 +289 val_289 +289 val_289 +291 val_291 +291 val_291 +292 val_292 +292 val_292 +296 val_296 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +30 val_30 +302 val_302 +302 val_302 +305 val_305 +305 val_305 +306 val_306 +306 val_306 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +308 val_308 +308 val_308 +309 val_309 +309 val_309 +309 val_309 +309 val_309 +310 val_310 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +315 val_315 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +323 val_323 +323 val_323 +325 val_325 +325 val_325 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +33 val_33 +331 val_331 +331 val_331 +331 val_331 +331 val_331 +332 val_332 +332 val_332 +333 val_333 +333 val_333 +333 val_333 +333 val_333 +335 val_335 +335 val_335 +336 val_336 +336 val_336 +338 val_338 +338 val_338 +339 val_339 +339 val_339 +34 val_34 +34 val_34 +341 val_341 +341 val_341 +342 val_342 +342 val_342 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +345 val_345 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +351 val_351 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +356 val_356 +356 val_356 +360 val_360 +360 val_360 +362 val_362 +362 val_362 +364 val_364 +364 val_364 +365 val_365 +365 val_365 +366 val_366 +366 val_366 +367 val_367 +367 val_367 +367 val_367 +367 val_367 +368 val_368 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +373 val_373 +373 val_373 +374 val_374 +374 val_374 +375 val_375 +375 val_375 +377 val_377 +377 val_377 +378 val_378 +378 val_378 +379 val_379 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +386 val_386 +389 val_389 +389 val_389 +392 val_392 +392 val_392 +393 val_393 +393 val_393 +394 val_394 +394 val_394 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +4 val_4 +4 val_4 +400 val_400 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 +402 val_402 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +404 val_404 +404 val_404 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +41 val_41 +411 val_411 +411 val_411 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +418 val_418 +419 val_419 +419 val_419 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +421 val_421 +421 val_421 +424 val_424 +424 val_424 +424 val_424 +424 val_424 +427 val_427 +427 val_427 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +43 val_43 +43 val_43 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +432 val_432 +432 val_432 +435 val_435 +435 val_435 +436 val_436 +436 val_436 +437 val_437 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +439 val_439 +439 val_439 +439 val_439 +439 val_439 +44 val_44 +44 val_44 +443 val_443 +443 val_443 +444 val_444 +444 val_444 +446 val_446 +446 val_446 +448 val_448 +448 val_448 +449 val_449 +449 val_449 +452 val_452 +452 val_452 +453 val_453 +453 val_453 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +455 val_455 +457 val_457 +457 val_457 +458 val_458 +458 val_458 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +460 val_460 +460 val_460 +462 val_462 +462 val_462 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +47 val_47 +470 val_470 +470 val_470 +472 val_472 +472 val_472 +475 val_475 +475 val_475 +477 val_477 +477 val_477 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +479 val_479 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +481 val_481 +482 val_482 +482 val_482 +483 val_483 +483 val_483 +484 val_484 +484 val_484 +485 val_485 +485 val_485 +487 val_487 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +490 val_490 +491 val_491 +491 val_491 +492 val_492 +492 val_492 +492 val_492 +492 val_492 +493 val_493 +493 val_493 +494 val_494 +494 val_494 +495 val_495 +495 val_495 +496 val_496 +496 val_496 +497 val_497 +497 val_497 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +53 val_53 +54 val_54 +54 val_54 +57 val_57 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +64 val_64 +65 val_65 +65 val_65 +66 val_66 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +77 val_77 +78 val_78 +78 val_78 +8 val_8 +8 val_8 +80 val_80 +80 val_80 +82 val_82 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +85 val_85 +85 val_85 +86 val_86 +86 val_86 +87 val_87 +87 val_87 +9 val_9 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +96 val_96 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 +PREHOOK: query: explain +select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +POSTHOOK: query: explain +select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: masking_test_n9 + filterExpr: (key > 0) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 0) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: int), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: masking_test_n9 + filterExpr: (key > 0) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 0) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: int), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col2), count(_col2) + keys: _col0 (type: int), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 27750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 250 Data size: 27750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), count(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 27750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col3 = 2L) (type: boolean) + Statistics: Num rows: 1 Data size: 111 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE + function name: UDTFReplicateRows + Select Operator + expressions: col1 (type: int), col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col2), count(_col2) + keys: _col0 (type: int), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 27750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 250 Data size: 27750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +POSTHOOK: query: select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_n9 +#### A masked pattern was here #### +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +126 val_126 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +134 val_134 +136 val_136 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +15 val_15 +15 val_15 +150 val_150 +152 val_152 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +166 val_166 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +191 val_191 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +195 val_195 +195 val_195 +196 val_196 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +203 val_203 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +208 val_208 +208 val_208 +208 val_208 +209 val_209 +209 val_209 +213 val_213 +213 val_213 +214 val_214 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +218 val_218 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +222 val_222 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +229 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +235 val_235 +237 val_237 +237 val_237 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +24 val_24 +24 val_24 +241 val_241 +242 val_242 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +255 val_255 +256 val_256 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +28 val_28 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +307 val_307 +308 val_308 +309 val_309 +309 val_309 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +323 val_323 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +331 val_331 +331 val_331 +332 val_332 +333 val_333 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +353 val_353 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +367 val_367 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +37 val_37 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +42 val_42 +421 val_421 +424 val_424 +424 val_424 +427 val_427 +429 val_429 +429 val_429 +43 val_43 +430 val_430 +430 val_430 +430 val_430 +431 val_431 +431 val_431 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +439 val_439 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +460 val_460 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +478 val_478 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +498 val_498 +498 val_498 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/llap/union_ppr.q.out ql/src/test/results/clientpositive/llap/union_ppr.q.out new file mode 100644 index 0000000000..940e3fb93f --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_ppr.q.out @@ -0,0 +1,709 @@ +PREHOOK: query: EXPLAIN EXTENDED +SELECT * FROM ( + SELECT X.* FROM SRCPART X WHERE X.key < 100 + UNION ALL + SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 +) A +WHERE A.ds = '2008-04-08' +SORT BY A.key, A.value, A.ds, A.hr +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN EXTENDED +SELECT * FROM ( + SELECT X.* FROM SRCPART X WHERE X.key < 100 + UNION ALL + SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 +) A +WHERE A.ds = '2008-04-08' +SORT BY A.key, A.value, A.ds, A.hr +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + filterExpr: ((UDFToDouble(key) < 100.0D) and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 100.0D) (type: boolean) + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + null sort order: zzz + numBuckets: -1 + sort order: +++ + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [x] + /srcpart/ds=2008-04-08/hr=12 [x] + Map 4 + Map Operator Tree: + TableScan + alias: y + filterExpr: ((UDFToDouble(key) < 100.0D) and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 100.0D) (type: boolean) + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + null sort order: zzz + numBuckets: -1 + sort order: +++ + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + auto parallelism: true + Execution mode: vectorized, llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [y] + /srcpart/ds=2008-04-08/hr=12 [y] + Reducer 3 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT X.* FROM SRCPART X WHERE X.key < 100 + UNION ALL + SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 +) A +WHERE A.ds = '2008-04-08' +SORT BY A.key, A.value, A.ds, A.hr +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT X.* FROM SRCPART X WHERE X.key < 100 + UNION ALL + SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 +) A +WHERE A.ds = '2008-04-08' +SORT BY A.key, A.value, A.ds, A.hr +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 11 +0 val_0 2008-04-08 12 +0 val_0 2008-04-08 12 +0 val_0 2008-04-08 12 +0 val_0 2008-04-08 12 +0 val_0 2008-04-08 12 +0 val_0 2008-04-08 12 +10 val_10 2008-04-08 11 +10 val_10 2008-04-08 11 +10 val_10 2008-04-08 12 +10 val_10 2008-04-08 12 +11 val_11 2008-04-08 11 +11 val_11 2008-04-08 11 +11 val_11 2008-04-08 12 +11 val_11 2008-04-08 12 +12 val_12 2008-04-08 11 +12 val_12 2008-04-08 11 +12 val_12 2008-04-08 11 +12 val_12 2008-04-08 11 +12 val_12 2008-04-08 12 +12 val_12 2008-04-08 12 +12 val_12 2008-04-08 12 +12 val_12 2008-04-08 12 +15 val_15 2008-04-08 11 +15 val_15 2008-04-08 11 +15 val_15 2008-04-08 11 +15 val_15 2008-04-08 11 +15 val_15 2008-04-08 12 +15 val_15 2008-04-08 12 +15 val_15 2008-04-08 12 +15 val_15 2008-04-08 12 +17 val_17 2008-04-08 11 +17 val_17 2008-04-08 11 +17 val_17 2008-04-08 12 +17 val_17 2008-04-08 12 +18 val_18 2008-04-08 11 +18 val_18 2008-04-08 11 +18 val_18 2008-04-08 11 +18 val_18 2008-04-08 11 +18 val_18 2008-04-08 12 +18 val_18 2008-04-08 12 +18 val_18 2008-04-08 12 +18 val_18 2008-04-08 12 +19 val_19 2008-04-08 11 +19 val_19 2008-04-08 11 +19 val_19 2008-04-08 12 +19 val_19 2008-04-08 12 +2 val_2 2008-04-08 11 +2 val_2 2008-04-08 11 +2 val_2 2008-04-08 12 +2 val_2 2008-04-08 12 +20 val_20 2008-04-08 11 +20 val_20 2008-04-08 11 +20 val_20 2008-04-08 12 +20 val_20 2008-04-08 12 +24 val_24 2008-04-08 11 +24 val_24 2008-04-08 11 +24 val_24 2008-04-08 11 +24 val_24 2008-04-08 11 +24 val_24 2008-04-08 12 +24 val_24 2008-04-08 12 +24 val_24 2008-04-08 12 +24 val_24 2008-04-08 12 +26 val_26 2008-04-08 11 +26 val_26 2008-04-08 11 +26 val_26 2008-04-08 11 +26 val_26 2008-04-08 11 +26 val_26 2008-04-08 12 +26 val_26 2008-04-08 12 +26 val_26 2008-04-08 12 +26 val_26 2008-04-08 12 +27 val_27 2008-04-08 11 +27 val_27 2008-04-08 11 +27 val_27 2008-04-08 12 +27 val_27 2008-04-08 12 +28 val_28 2008-04-08 11 +28 val_28 2008-04-08 11 +28 val_28 2008-04-08 12 +28 val_28 2008-04-08 12 +30 val_30 2008-04-08 11 +30 val_30 2008-04-08 11 +30 val_30 2008-04-08 12 +30 val_30 2008-04-08 12 +33 val_33 2008-04-08 11 +33 val_33 2008-04-08 11 +33 val_33 2008-04-08 12 +33 val_33 2008-04-08 12 +34 val_34 2008-04-08 11 +34 val_34 2008-04-08 11 +34 val_34 2008-04-08 12 +34 val_34 2008-04-08 12 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 11 +35 val_35 2008-04-08 12 +35 val_35 2008-04-08 12 +35 val_35 2008-04-08 12 +35 val_35 2008-04-08 12 +35 val_35 2008-04-08 12 +35 val_35 2008-04-08 12 +37 val_37 2008-04-08 11 +37 val_37 2008-04-08 11 +37 val_37 2008-04-08 11 +37 val_37 2008-04-08 11 +37 val_37 2008-04-08 12 +37 val_37 2008-04-08 12 +37 val_37 2008-04-08 12 +37 val_37 2008-04-08 12 +4 val_4 2008-04-08 11 +4 val_4 2008-04-08 11 +4 val_4 2008-04-08 12 +4 val_4 2008-04-08 12 +41 val_41 2008-04-08 11 +41 val_41 2008-04-08 11 +41 val_41 2008-04-08 12 +41 val_41 2008-04-08 12 +42 val_42 2008-04-08 11 +42 val_42 2008-04-08 11 +42 val_42 2008-04-08 11 +42 val_42 2008-04-08 11 +42 val_42 2008-04-08 12 +42 val_42 2008-04-08 12 +42 val_42 2008-04-08 12 +42 val_42 2008-04-08 12 +43 val_43 2008-04-08 11 +43 val_43 2008-04-08 11 +43 val_43 2008-04-08 12 +43 val_43 2008-04-08 12 +44 val_44 2008-04-08 11 +44 val_44 2008-04-08 11 +44 val_44 2008-04-08 12 +44 val_44 2008-04-08 12 +47 val_47 2008-04-08 11 +47 val_47 2008-04-08 11 +47 val_47 2008-04-08 12 +47 val_47 2008-04-08 12 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +51 val_51 2008-04-08 11 +51 val_51 2008-04-08 11 +51 val_51 2008-04-08 11 +51 val_51 2008-04-08 11 +51 val_51 2008-04-08 12 +51 val_51 2008-04-08 12 +51 val_51 2008-04-08 12 +51 val_51 2008-04-08 12 +53 val_53 2008-04-08 11 +53 val_53 2008-04-08 11 +53 val_53 2008-04-08 12 +53 val_53 2008-04-08 12 +54 val_54 2008-04-08 11 +54 val_54 2008-04-08 11 +54 val_54 2008-04-08 12 +54 val_54 2008-04-08 12 +57 val_57 2008-04-08 11 +57 val_57 2008-04-08 11 +57 val_57 2008-04-08 12 +57 val_57 2008-04-08 12 +58 val_58 2008-04-08 11 +58 val_58 2008-04-08 11 +58 val_58 2008-04-08 11 +58 val_58 2008-04-08 11 +58 val_58 2008-04-08 12 +58 val_58 2008-04-08 12 +58 val_58 2008-04-08 12 +58 val_58 2008-04-08 12 +64 val_64 2008-04-08 11 +64 val_64 2008-04-08 11 +64 val_64 2008-04-08 12 +64 val_64 2008-04-08 12 +65 val_65 2008-04-08 11 +65 val_65 2008-04-08 11 +65 val_65 2008-04-08 12 +65 val_65 2008-04-08 12 +66 val_66 2008-04-08 11 +66 val_66 2008-04-08 11 +66 val_66 2008-04-08 12 +66 val_66 2008-04-08 12 +67 val_67 2008-04-08 11 +67 val_67 2008-04-08 11 +67 val_67 2008-04-08 11 +67 val_67 2008-04-08 11 +67 val_67 2008-04-08 12 +67 val_67 2008-04-08 12 +67 val_67 2008-04-08 12 +67 val_67 2008-04-08 12 +69 val_69 2008-04-08 11 +69 val_69 2008-04-08 11 +69 val_69 2008-04-08 12 +69 val_69 2008-04-08 12 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 11 +70 val_70 2008-04-08 12 +70 val_70 2008-04-08 12 +70 val_70 2008-04-08 12 +70 val_70 2008-04-08 12 +70 val_70 2008-04-08 12 +70 val_70 2008-04-08 12 +72 val_72 2008-04-08 11 +72 val_72 2008-04-08 11 +72 val_72 2008-04-08 11 +72 val_72 2008-04-08 11 +72 val_72 2008-04-08 12 +72 val_72 2008-04-08 12 +72 val_72 2008-04-08 12 +72 val_72 2008-04-08 12 +74 val_74 2008-04-08 11 +74 val_74 2008-04-08 11 +74 val_74 2008-04-08 12 +74 val_74 2008-04-08 12 +76 val_76 2008-04-08 11 +76 val_76 2008-04-08 11 +76 val_76 2008-04-08 11 +76 val_76 2008-04-08 11 +76 val_76 2008-04-08 12 +76 val_76 2008-04-08 12 +76 val_76 2008-04-08 12 +76 val_76 2008-04-08 12 +77 val_77 2008-04-08 11 +77 val_77 2008-04-08 11 +77 val_77 2008-04-08 12 +77 val_77 2008-04-08 12 +78 val_78 2008-04-08 11 +78 val_78 2008-04-08 11 +78 val_78 2008-04-08 12 +78 val_78 2008-04-08 12 +8 val_8 2008-04-08 11 +8 val_8 2008-04-08 11 +8 val_8 2008-04-08 12 +8 val_8 2008-04-08 12 +80 val_80 2008-04-08 11 +80 val_80 2008-04-08 11 +80 val_80 2008-04-08 12 +80 val_80 2008-04-08 12 +82 val_82 2008-04-08 11 +82 val_82 2008-04-08 11 +82 val_82 2008-04-08 12 +82 val_82 2008-04-08 12 +83 val_83 2008-04-08 11 +83 val_83 2008-04-08 11 +83 val_83 2008-04-08 11 +83 val_83 2008-04-08 11 +83 val_83 2008-04-08 12 +83 val_83 2008-04-08 12 +83 val_83 2008-04-08 12 +83 val_83 2008-04-08 12 +84 val_84 2008-04-08 11 +84 val_84 2008-04-08 11 +84 val_84 2008-04-08 11 +84 val_84 2008-04-08 11 +84 val_84 2008-04-08 12 +84 val_84 2008-04-08 12 +84 val_84 2008-04-08 12 +84 val_84 2008-04-08 12 +85 val_85 2008-04-08 11 +85 val_85 2008-04-08 11 +85 val_85 2008-04-08 12 +85 val_85 2008-04-08 12 +86 val_86 2008-04-08 11 +86 val_86 2008-04-08 11 +86 val_86 2008-04-08 12 +86 val_86 2008-04-08 12 +87 val_87 2008-04-08 11 +87 val_87 2008-04-08 11 +87 val_87 2008-04-08 12 +87 val_87 2008-04-08 12 +9 val_9 2008-04-08 11 +9 val_9 2008-04-08 11 +9 val_9 2008-04-08 12 +9 val_9 2008-04-08 12 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 11 +90 val_90 2008-04-08 12 +90 val_90 2008-04-08 12 +90 val_90 2008-04-08 12 +90 val_90 2008-04-08 12 +90 val_90 2008-04-08 12 +90 val_90 2008-04-08 12 +92 val_92 2008-04-08 11 +92 val_92 2008-04-08 11 +92 val_92 2008-04-08 12 +92 val_92 2008-04-08 12 +95 val_95 2008-04-08 11 +95 val_95 2008-04-08 11 +95 val_95 2008-04-08 11 +95 val_95 2008-04-08 11 +95 val_95 2008-04-08 12 +95 val_95 2008-04-08 12 +95 val_95 2008-04-08 12 +95 val_95 2008-04-08 12 +96 val_96 2008-04-08 11 +96 val_96 2008-04-08 11 +96 val_96 2008-04-08 12 +96 val_96 2008-04-08 12 +97 val_97 2008-04-08 11 +97 val_97 2008-04-08 11 +97 val_97 2008-04-08 11 +97 val_97 2008-04-08 11 +97 val_97 2008-04-08 12 +97 val_97 2008-04-08 12 +97 val_97 2008-04-08 12 +97 val_97 2008-04-08 12 +98 val_98 2008-04-08 11 +98 val_98 2008-04-08 11 +98 val_98 2008-04-08 11 +98 val_98 2008-04-08 11 +98 val_98 2008-04-08 12 +98 val_98 2008-04-08 12 +98 val_98 2008-04-08 12 +98 val_98 2008-04-08 12 diff --git ql/src/test/results/clientpositive/union_remove_1.q.out ql/src/test/results/clientpositive/llap/union_remove_1.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union_remove_1.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_1.q.out index dda4674044..bc4c56a840 100644 --- ql/src/test/results/clientpositive/union_remove_1.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_1.q.out @@ -46,50 +46,70 @@ POSTHOOK: Input: default@inputtbl1 POSTHOOK: Output: default@outputtbl1 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -101,47 +121,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -182,7 +161,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 40 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_10.q.out ql/src/test/results/clientpositive/llap/union_remove_10.q.out similarity index 54% rename from ql/src/test/results/clientpositive/union_remove_10.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_10.q.out index 6002274420..a12951fd46 100644 --- ql/src/test/results/clientpositive/union_remove_10.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_10.q.out @@ -54,44 +54,86 @@ POSTHOOK: Input: default@inputtbl1_n7 POSTHOOK: Output: default@outputtbl1_n9 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage - Stage-8 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n7 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n7 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n9 + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Select Operator + expressions: key (type: string), 2L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n9 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n9 - Execution mode: vectorized - - Stage: Stage-6 - Conditional Operator + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n9 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -103,87 +145,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n9 - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n7 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n9 - - Stage: Stage-8 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n7 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n9 - Execution mode: vectorized - PREHOOK: query: insert overwrite table outputTbl1_n9 SELECT * FROM ( @@ -232,8 +193,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 3 - totalSize 271 + numFiles 4 + totalSize 350 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_11.q.out ql/src/test/results/clientpositive/llap/union_remove_11.q.out similarity index 53% rename from ql/src/test/results/clientpositive/union_remove_11.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_11.q.out index a66e963a45..a06ff2175a 100644 --- ql/src/test/results/clientpositive/union_remove_11.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_11.q.out @@ -54,89 +54,93 @@ POSTHOOK: Input: default@inputtbl1_n14 POSTHOOK: Output: default@outputtbl1_n21 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n14 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n21 - TableScan - alias: inputtbl1_n14 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n21 - TableScan - alias: inputtbl1_n14 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n21 - - Stage: Stage-6 - Conditional Operator - - Stage: Stage-3 - Move Operator - files: - hdfs directory: true + Tez #### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n14 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n21 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: inputtbl1_n14 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n21 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: inputtbl1_n14 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 3 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n21 + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -148,26 +152,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n21 - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table outputTbl1_n21 SELECT * FROM ( @@ -216,8 +200,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 1 - totalSize 115 + numFiles 3 + totalSize 273 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_12.q.out ql/src/test/results/clientpositive/llap/union_remove_12.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union_remove_12.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_12.q.out index 548a514bf1..7e2500ab37 100644 --- ql/src/test/results/clientpositive/union_remove_12.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_12.q.out @@ -48,136 +48,99 @@ POSTHOOK: Input: default@inputtbl1_n21 POSTHOOK: Output: default@outputtbl1_n29 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n21 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n29 - Execution mode: vectorized - - Stage: Stage-6 - Conditional Operator - - Stage: Stage-3 - Move Operator - files: - hdfs directory: true + Tez #### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n29 - - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) #### A masked pattern was here #### - - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0-subquery2:$hdt$_0:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0-subquery2:$hdt$_0:a - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), UDFToLong(val) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n21 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) + expressions: key (type: string), 1L (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n29 - Execution mode: vectorized - Local Work: - Map Reduce Local Work + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), UDFToLong(val) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + input vertices: + 0 Map 1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n29 + Execution mode: vectorized, llap + LLAP IO: no inputs + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n29 PREHOOK: query: insert overwrite table outputTbl1_n29 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_13.q.out ql/src/test/results/clientpositive/llap/union_remove_13.q.out similarity index 50% rename from ql/src/test/results/clientpositive/union_remove_13.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_13.q.out index 797dcd4749..29d25627e3 100644 --- ql/src/test/results/clientpositive/union_remove_13.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_13.q.out @@ -48,65 +48,113 @@ POSTHOOK: Input: default@inputtbl1_n2 POSTHOOK: Output: default@outputtbl1_n3 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n2 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), UDFToLong(val) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + input vertices: + 0 Map 1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n3 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n3 - - Stage: Stage-6 - Conditional Operator + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n3 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -118,88 +166,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n3 - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0-subquery2:$hdt$_0:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0-subquery2:$hdt$_0:a - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), UDFToLong(val) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n3 - Execution mode: vectorized - Local Work: - Map Reduce Local Work - PREHOOK: query: insert overwrite table outputTbl1_n3 SELECT * FROM ( @@ -242,8 +208,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 - totalSize 192 + numFiles 3 + totalSize 271 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_14.q.out ql/src/test/results/clientpositive/llap/union_remove_14.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union_remove_14.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_14.q.out index 528a2b2fc4..09bb92a7a5 100644 --- ql/src/test/results/clientpositive/union_remove_14.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_14.q.out @@ -48,136 +48,99 @@ POSTHOOK: Input: default@inputtbl1_n11 POSTHOOK: Output: default@outputtbl1_n16 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n11 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n16 - Execution mode: vectorized - - Stage: Stage-6 - Conditional Operator - - Stage: Stage-3 - Move Operator - files: - hdfs directory: true + Tez #### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n16 - - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) #### A masked pattern was here #### - - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0-subquery2:$hdt$_0:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0-subquery2:$hdt$_0:a - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), UDFToLong(val) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n11 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) + expressions: key (type: string), 1L (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n16 - Execution mode: vectorized - Local Work: - Map Reduce Local Work + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 2 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), UDFToLong(val) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + input vertices: + 0 Map 1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n16 + Execution mode: vectorized, llap + LLAP IO: no inputs + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n16 PREHOOK: query: insert overwrite table outputTbl1_n16 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_15.q.out ql/src/test/results/clientpositive/llap/union_remove_15.q.out similarity index 61% rename from ql/src/test/results/clientpositive/union_remove_15.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_15.q.out index af476e9805..b02938da01 100644 --- ql/src/test/results/clientpositive/union_remove_15.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_15.q.out @@ -45,54 +45,78 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1_n18 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n18 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n18 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n25 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n25 + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n25 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,51 +130,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n25 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n18 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n25 - PREHOOK: query: insert overwrite table outputTbl1_n25 partition (ds) SELECT * FROM ( @@ -198,11 +177,11 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 numPartitions 2 numRows 0 rawDataSize 0 - totalSize 178 + totalSize 336 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_16.q.out ql/src/test/results/clientpositive/llap/union_remove_16.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union_remove_16.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_16.q.out index 337d90b9ab..6d715f2fa5 100644 --- ql/src/test/results/clientpositive/union_remove_16.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_16.q.out @@ -45,68 +45,78 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1_n23 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n23 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n23 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n32 - - Stage: Stage-6 - Conditional Operator + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n32 + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n32 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -120,71 +130,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n32 - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n23 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n32 - PREHOOK: query: insert overwrite table outputTbl1_n32 partition (ds) SELECT * FROM ( @@ -232,11 +177,11 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 numPartitions 2 numRows 0 rawDataSize 0 - totalSize 178 + totalSize 336 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_17.q.out ql/src/test/results/clientpositive/llap/union_remove_17.q.out similarity index 63% rename from ql/src/test/results/clientpositive/union_remove_17.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_17.q.out index 43f0662ffa..48b60f8a09 100644 --- ql/src/test/results/clientpositive/union_remove_17.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_17.q.out @@ -45,65 +45,83 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1_n3 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n3 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1 (type: int), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) - TableScan - alias: inputtbl1_n3 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2 (type: int), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n3 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: inputtbl1_n3 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 2 (type: int), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), KEY._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n4 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n4 + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/union_remove_18.q.out ql/src/test/results/clientpositive/llap/union_remove_18.q.out similarity index 67% rename from ql/src/test/results/clientpositive/union_remove_18.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_18.q.out index 9894a437db..93b77f3baf 100644 --- ql/src/test/results/clientpositive/union_remove_18.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_18.q.out @@ -45,54 +45,74 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1_n22 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n22 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), ds (type: string) - outputColumnNames: key, ds - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n22 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), ds (type: string) + outputColumnNames: key, ds + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string), ds (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string), ds (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n30 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n30 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n30 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,51 +126,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n30 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n22 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), ds (type: string) - outputColumnNames: key, ds - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string), ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n30 - PREHOOK: query: insert overwrite table outputTbl1_n30 partition (ds) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/llap/union_remove_19.q.out ql/src/test/results/clientpositive/llap/union_remove_19.q.out new file mode 100644 index 0000000000..8d66038364 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_remove_19.q.out @@ -0,0 +1,475 @@ +PREHOOK: query: create table inputTbl1_n1(key string, val string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@inputTbl1_n1 +POSTHOOK: query: create table inputTbl1_n1(key string, val string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@inputTbl1_n1 +PREHOOK: query: create table outputTbl1_n1(key string, `values` bigint) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@outputTbl1_n1 +POSTHOOK: query: create table outputTbl1_n1(key string, `values` bigint) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@outputTbl1_n1 +PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@inputtbl1_n1 +POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@inputtbl1_n1 +PREHOOK: query: explain +insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: explain +insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + +PREHOOK: query: insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] +PREHOOK: query: desc formatted outputTbl1_n1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@outputtbl1_n1 +POSTHOOK: query: desc formatted outputTbl1_n1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@outputtbl1_n1 +# col_name data_type comment +key string +values bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + bucketing_version 2 + numFiles 4 + totalSize 40 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from outputTbl1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +1 1 +1 1 +2 1 +2 1 +3 1 +3 1 +7 1 +7 1 +8 2 +8 2 +PREHOOK: query: explain +insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a where a.key = 7 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: explain +insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a where a.key = 7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n1 + filterExpr: (UDFToDouble(key) = 7.0D) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 7.0D) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + +PREHOOK: query: insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a where a.key = 7 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: insert overwrite table outputTbl1_n1 +SELECT a.key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a where a.key = 7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] +PREHOOK: query: select * from outputTbl1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +7 1 +7 1 +PREHOOK: query: explain +insert overwrite table outputTbl1_n1 +select key, `values` from +( +SELECT a.key + a.key as key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +) b where b.key >= 7 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: explain +insert overwrite table outputTbl1_n1 +select key, `values` from +( +SELECT a.key + a.key as key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +) b where b.key >= 7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n1 + filterExpr: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: CAST( (UDFToDouble(_col0) + UDFToDouble(_col0)) AS STRING) (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n1 + +PREHOOK: query: insert overwrite table outputTbl1_n1 +select key, `values` from +( +SELECT a.key + a.key as key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +) b where b.key >= 7 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n1 +PREHOOK: Output: default@outputtbl1_n1 +POSTHOOK: query: insert overwrite table outputTbl1_n1 +select key, `values` from +( +SELECT a.key + a.key as key, a.`values` +FROM ( + SELECT key, count(1) as `values` from inputTbl1_n1 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n1 group by key +) a +) b where b.key >= 7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n1 +POSTHOOK: Output: default@outputtbl1_n1 +POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] +PREHOOK: query: select * from outputTbl1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl1_n1 +#### A masked pattern was here #### +14.0 1 +14.0 1 +16.0 2 +16.0 2 diff --git ql/src/test/results/clientpositive/union_remove_2.q.out ql/src/test/results/clientpositive/llap/union_remove_2.q.out similarity index 57% rename from ql/src/test/results/clientpositive/union_remove_2.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_2.q.out index 00b51385c4..9b544e8938 100644 --- ql/src/test/results/clientpositive/union_remove_2.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_2.q.out @@ -50,51 +50,86 @@ POSTHOOK: Input: default@inputtbl1_n8 POSTHOOK: Output: default@outputtbl1_n11 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage - Stage-3 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n8 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n8 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Select Operator + expressions: key (type: string), 1L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n11 + Select Operator + expressions: key (type: string), 2L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n11 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n11 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n11 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,46 +141,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n11 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n8 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n11 - Execution mode: vectorized - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n8 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n11 - Execution mode: vectorized - PREHOOK: query: insert overwrite table outputTbl1_n11 SELECT * FROM ( @@ -190,7 +185,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 3 + numFiles 4 totalSize 68 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_20.q.out ql/src/test/results/clientpositive/llap/union_remove_20.q.out similarity index 56% rename from ql/src/test/results/clientpositive/union_remove_20.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_20.q.out index b95f263d27..c45d6ebfea 100644 --- ql/src/test/results/clientpositive/union_remove_20.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_20.q.out @@ -46,54 +46,74 @@ POSTHOOK: Input: default@inputtbl1_n19 POSTHOOK: Output: default@outputtbl1_n27 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n19 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n19 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col1 (type: bigint), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n27 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n27 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n27 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -105,51 +125,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n27 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n19 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n27 - PREHOOK: query: insert overwrite table outputTbl1_n27 SELECT a.`values`, a.key FROM ( @@ -190,7 +165,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 40 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_21.q.out ql/src/test/results/clientpositive/llap/union_remove_21.q.out similarity index 60% rename from ql/src/test/results/clientpositive/union_remove_21.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_21.q.out index 4f2c4d0b01..f5b010bd42 100644 --- ql/src/test/results/clientpositive/union_remove_21.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_21.q.out @@ -46,47 +46,67 @@ POSTHOOK: Input: default@inputtbl1_n12 POSTHOOK: Output: default@outputtbl1_n17 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n12 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n12 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n17 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n17 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n17 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -98,44 +118,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n17 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n12 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n17 - PREHOOK: query: insert overwrite table outputTbl1_n17 SELECT a.key FROM ( @@ -174,7 +156,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 20 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_22.q.out ql/src/test/results/clientpositive/llap/union_remove_22.q.out similarity index 50% rename from ql/src/test/results/clientpositive/union_remove_22.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_22.q.out index 8b4935d410..6abdb50273 100644 --- ql/src/test/results/clientpositive/union_remove_22.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_22.q.out @@ -46,54 +46,74 @@ POSTHOOK: Input: default@inputtbl1_n5 POSTHOOK: Output: default@outputtbl1_n7 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n5 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n5 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n7 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n7 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n7 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -105,51 +125,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n7 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n5 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n7 - PREHOOK: query: insert overwrite table outputTbl1_n7 SELECT a.key, a.`values`, a.`values` FROM ( @@ -192,7 +167,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 60 #### A masked pattern was here #### @@ -248,54 +223,74 @@ POSTHOOK: Input: default@inputtbl1_n5 POSTHOOK: Output: default@outputtbl1_n7 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n5 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n5 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n7 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n7 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n7 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -307,51 +302,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n7 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n5 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint), UDFToLong(concat(CAST( _col1 AS STRING), _col1)) (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n7 - PREHOOK: query: insert overwrite table outputTbl1_n7 SELECT a.key, concat(a.`values`, a.`values`), concat(a.`values`, a.`values`) FROM ( diff --git ql/src/test/results/clientpositive/llap/union_remove_23.q.out ql/src/test/results/clientpositive/llap/union_remove_23.q.out new file mode 100644 index 0000000000..53c1ad3039 --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_remove_23.q.out @@ -0,0 +1,269 @@ +PREHOOK: query: create table inputTbl1_n25(key string, val string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@inputTbl1_n25 +POSTHOOK: query: create table inputTbl1_n25(key string, val string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@inputTbl1_n25 +PREHOOK: query: create table outputTbl1_n34(key string, `values` bigint) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@outputTbl1_n34 +POSTHOOK: query: create table outputTbl1_n34(key string, `values` bigint) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@outputTbl1_n34 +PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n25 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@inputtbl1_n25 +POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n25 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@inputtbl1_n25 +PREHOOK: query: explain +insert overwrite table outputTbl1_n34 +SELECT * +FROM ( + SELECT key, count(1) as `values` from + (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n25 group by key +) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n25 +PREHOOK: Output: default@outputtbl1_n34 +POSTHOOK: query: explain +insert overwrite table outputTbl1_n34 +SELECT * +FROM ( + SELECT key, count(1) as `values` from + (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n25 group by key +) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n25 +POSTHOOK: Output: default@outputtbl1_n34 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n34 + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n34 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n34 + +PREHOOK: query: insert overwrite table outputTbl1_n34 +SELECT * +FROM ( + SELECT key, count(1) as `values` from + (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n25 group by key +) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n25 +PREHOOK: Output: default@outputtbl1_n34 +POSTHOOK: query: insert overwrite table outputTbl1_n34 +SELECT * +FROM ( + SELECT key, count(1) as `values` from + (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n25 group by key +) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n25 +POSTHOOK: Output: default@outputtbl1_n34 +POSTHOOK: Lineage: outputtbl1_n34.key EXPRESSION [(inputtbl1_n25)a.FieldSchema(name:key, type:string, comment:null), (inputtbl1_n25)inputtbl1_n25.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl1_n34.values EXPRESSION [(inputtbl1_n25)a.null, (inputtbl1_n25)b.null, (inputtbl1_n25)inputtbl1_n25.null, ] +PREHOOK: query: desc formatted outputTbl1_n34 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@outputtbl1_n34 +POSTHOOK: query: desc formatted outputTbl1_n34 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@outputtbl1_n34 +# col_name data_type comment +key string +values bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + bucketing_version 2 + numFiles 4 + totalSize 40 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from outputTbl1_n34 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl1_n34 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl1_n34 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl1_n34 +#### A masked pattern was here #### +1 1 +1 1 +2 1 +2 1 +3 1 +3 1 +7 1 +7 1 +8 2 +8 4 diff --git ql/src/test/results/clientpositive/union_remove_24.q.out ql/src/test/results/clientpositive/llap/union_remove_24.q.out similarity index 56% rename from ql/src/test/results/clientpositive/union_remove_24.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_24.q.out index 085f910f98..9d9413adc7 100644 --- ql/src/test/results/clientpositive/union_remove_24.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_24.q.out @@ -46,54 +46,78 @@ POSTHOOK: Input: default@inputtbl1_n20 POSTHOOK: Output: default@outputtbl1_n28 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n20 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n20 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: UDFToDouble(_col0) (type: double), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(_col0) (type: double), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n28 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n28 + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n28 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -105,51 +129,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n28 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n20 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n28 - PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1_n28 SELECT * FROM ( @@ -190,7 +169,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 60 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_25.q.out ql/src/test/results/clientpositive/llap/union_remove_25.q.out similarity index 57% rename from ql/src/test/results/clientpositive/union_remove_25.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_25.q.out index 6a324e5e9d..1cbc43b2fe 100644 --- ql/src/test/results/clientpositive/union_remove_25.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_25.q.out @@ -62,50 +62,70 @@ POSTHOOK: Input: default@inputtbl1_n13 POSTHOOK: Output: default@outputtbl1_n19@ds=2004 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n13 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n13 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n19 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n19 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n19 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -119,47 +139,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n19 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n13 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n19 - PREHOOK: query: insert overwrite table outputTbl1_n19 partition(ds='2004') SELECT * FROM ( @@ -202,7 +181,7 @@ Database: default Table: outputtbl1_n19 #### A masked pattern was here #### Partition Parameters: - numFiles 2 + numFiles 4 totalSize 40 #### A masked pattern was here #### @@ -263,51 +242,71 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 500 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 500 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: + Limit + Number of rows: 500 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 500 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2_n6 + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2_n6 + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2_n6 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -321,48 +320,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2_n6 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 500 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 500 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 94500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2_n6 - PREHOOK: query: insert overwrite table outputTbl2_n6 partition(ds) SELECT * FROM ( @@ -455,51 +412,71 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1000 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1000 + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: + Limit + Number of rows: 1000 Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1000 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl3_n3 + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl3_n3 + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl3_n3 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -514,48 +491,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl3_n3 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1000 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1000 - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl3_n3 - PREHOOK: query: insert overwrite table outputTbl3_n3 partition(ds, hr) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_3.q.out ql/src/test/results/clientpositive/llap/union_remove_3.q.out similarity index 53% rename from ql/src/test/results/clientpositive/union_remove_3.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_3.q.out index cd8899ea49..b7a60c4984 100644 --- ql/src/test/results/clientpositive/union_remove_3.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_3.q.out @@ -50,75 +50,93 @@ POSTHOOK: Input: default@inputtbl1_n16 POSTHOOK: Output: default@outputtbl1_n23 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n16 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n23 - TableScan - alias: inputtbl1_n16 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n23 - TableScan - alias: inputtbl1_n16 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n23 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n16 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n23 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: inputtbl1_n16 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n23 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: inputtbl1_n16 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 3 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n23 + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -174,7 +192,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 1 + numFiles 3 totalSize 72 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_4.q.out ql/src/test/results/clientpositive/llap/union_remove_4.q.out similarity index 51% rename from ql/src/test/results/clientpositive/union_remove_4.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_4.q.out index ed1c145902..95f3f5b19f 100644 --- ql/src/test/results/clientpositive/union_remove_4.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_4.q.out @@ -46,64 +46,70 @@ POSTHOOK: Input: default@inputtbl1_n24 POSTHOOK: Output: default@outputtbl1_n33 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n24 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n24 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n33 - - Stage: Stage-6 - Conditional Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n33 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n33 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -115,77 +121,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n33 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n33 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n33 - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n24 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n33 - PREHOOK: query: insert overwrite table outputTbl1_n33 SELECT * FROM ( @@ -226,7 +161,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 + numFiles 4 totalSize 40 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_5.q.out ql/src/test/results/clientpositive/llap/union_remove_5.q.out similarity index 51% rename from ql/src/test/results/clientpositive/union_remove_5.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_5.q.out index 92a24e0dd9..2f34d978a5 100644 --- ql/src/test/results/clientpositive/union_remove_5.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_5.q.out @@ -50,65 +50,86 @@ POSTHOOK: Input: default@inputtbl1_n4 POSTHOOK: Output: default@outputtbl1_n6 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage - Stage-8 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n4 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n4 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Select Operator + expressions: key (type: string), 1L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n6 + Select Operator + expressions: key (type: string), 2L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n6 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n6 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n6 - Stage: Stage-6 - Conditional Operator - - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -120,76 +141,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1_n6 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n6 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n6 - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n4 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n6 - Execution mode: vectorized - - Stage: Stage-8 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n4 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n6 - Execution mode: vectorized - PREHOOK: query: insert overwrite table outputTbl1_n6 SELECT * FROM ( @@ -234,7 +185,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 3 + numFiles 4 totalSize 68 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/union_remove_6.q.out ql/src/test/results/clientpositive/llap/union_remove_6.q.out similarity index 52% rename from ql/src/test/results/clientpositive/union_remove_6.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_6.q.out index 407bb3336c..99db987271 100644 --- ql/src/test/results/clientpositive/union_remove_6.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_6.q.out @@ -56,92 +56,124 @@ POSTHOOK: Output: default@outputtbl1_n14 POSTHOOK: Output: default@outputtbl2_n4 STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage STAGE PLANS: Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n10 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n10 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: inputtbl1_n10 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n14 + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2_n4 + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n14 + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2_n4 + Union 3 + Vertex: Union 3 Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n14 - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2_n4 - TableScan - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n14 - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2_n4 + Dependency Collection Stage: Stage-0 Move Operator @@ -163,45 +195,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2_n4 - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n10 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - PREHOOK: query: FROM ( SELECT key, count(1) as `values` from inputTbl1_n10 group by key UNION ALL diff --git ql/src/test/results/clientpositive/llap/union_remove_6_subq.q.out ql/src/test/results/clientpositive/llap/union_remove_6_subq.q.out new file mode 100644 index 0000000000..fb6ec1131c --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_remove_6_subq.q.out @@ -0,0 +1,1243 @@ +PREHOOK: query: create table inputTbl1_n0(key string, val string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@inputTbl1_n0 +POSTHOOK: query: create table inputTbl1_n0(key string, val string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@inputTbl1_n0 +PREHOOK: query: create table outputTbl1_n0(key string, `values` bigint) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@outputTbl1_n0 +POSTHOOK: query: create table outputTbl1_n0(key string, `values` bigint) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@outputTbl1_n0 +PREHOOK: query: create table outputTbl2(key string, `values` bigint) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@outputTbl2 +POSTHOOK: query: create table outputTbl2(key string, `values` bigint) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@outputTbl2 +PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n0 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@inputtbl1_n0 +POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n0 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@inputtbl1_n0 +PREHOOK: query: explain +FROM ( + select * from( + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + )subq +) a +insert overwrite table outputTbl1_n0 select * +insert overwrite table outputTbl2 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n0 +PREHOOK: Output: default@outputtbl1_n0 +PREHOOK: Output: default@outputtbl2 +POSTHOOK: query: explain +FROM ( + select * from( + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + )subq +) a +insert overwrite table outputTbl1_n0 select * +insert overwrite table outputTbl2 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n0 +POSTHOOK: Output: default@outputtbl1_n0 +POSTHOOK: Output: default@outputtbl2 +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: inputtbl1_n0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n0 + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2 + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n0 + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1_n0 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl2 + +PREHOOK: query: FROM ( + select * from( + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + )subq +) a +insert overwrite table outputTbl1_n0 select * +insert overwrite table outputTbl2 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@inputtbl1_n0 +PREHOOK: Output: default@outputtbl1_n0 +PREHOOK: Output: default@outputtbl2 +POSTHOOK: query: FROM ( + select * from( + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + UNION ALL + SELECT key, count(1) as `values` from inputTbl1_n0 group by key + )subq +) a +insert overwrite table outputTbl1_n0 select * +insert overwrite table outputTbl2 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@inputtbl1_n0 +POSTHOOK: Output: default@outputtbl1_n0 +POSTHOOK: Output: default@outputtbl2 +POSTHOOK: Lineage: outputtbl1_n0.key EXPRESSION [(inputtbl1_n0)inputtbl1_n0.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl1_n0.values EXPRESSION [(inputtbl1_n0)inputtbl1_n0.null, ] +POSTHOOK: Lineage: outputtbl2.key EXPRESSION [(inputtbl1_n0)inputtbl1_n0.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: outputtbl2.values EXPRESSION [(inputtbl1_n0)inputtbl1_n0.null, ] +PREHOOK: query: select * from outputTbl1_n0 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl1_n0 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl1_n0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl1_n0 +#### A masked pattern was here #### +1 1 +1 1 +2 1 +2 1 +3 1 +3 1 +7 1 +7 1 +8 2 +8 2 +PREHOOK: query: select * from outputTbl2 +PREHOOK: type: QUERY +PREHOOK: Input: default@outputtbl2 +#### A masked pattern was here #### +POSTHOOK: query: select * from outputTbl2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@outputtbl2 +#### A masked pattern was here #### +1 1 +1 1 +2 1 +2 1 +3 1 +3 1 +7 1 +7 1 +8 2 +8 2 +PREHOOK: query: explain +select avg(c) from( + SELECT count(1)-200 as c from src + UNION ALL + SELECT count(1) as c from src +)subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain +select avg(c) from( + SELECT count(1)-200 as c from src + UNION ALL + SELECT count(1) as c from src +)subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (_col0 - 200L) (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0), count(_col0) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (UDFToDouble(_col0) / _col1) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0), count(_col0) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select avg(c) from( + SELECT count(1)-200 as c from src + UNION ALL + SELECT count(1) as c from src +)subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select avg(c) from( + SELECT count(1)-200 as c from src + UNION ALL + SELECT count(1) as c from src +)subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +400.0 +PREHOOK: query: explain +select key, avg(c) over w from( + SELECT key, count(1)*2 as c from src group by key + UNION ALL + SELECT key, count(1) as c from src group by key +)subq group by key, c +WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: explain +select key, avg(c) over w from( + SELECT key, count(1)*2 as c from src group by key + UNION ALL + SELECT key, count(1) as c from src group by key +)subq group by key, c +WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), (_col1 * 2L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: avg_window_0 + arguments: _col1 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: ROWS PRECEDING(MAX)~CURRENT + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), avg_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, avg(c) over w from( + SELECT key, count(1)*2 as c from src group by key + UNION ALL + SELECT key, count(1) as c from src group by key +)subq group by key, c +WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select key, avg(c) over w from( + SELECT key, count(1)*2 as c from src group by key + UNION ALL + SELECT key, count(1) as c from src group by key +)subq group by key, c +WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 3.0 +0 4.5 +10 1.0 +10 1.5 +100 2.0 +100 3.0 +103 2.0 +103 3.0 +104 2.0 +104 3.0 +105 1.0 +105 1.5 +11 1.0 +11 1.5 +111 1.0 +111 1.5 +113 2.0 +113 3.0 +114 1.0 +114 1.5 +116 1.0 +116 1.5 +118 2.0 +118 3.0 +119 3.0 +119 4.5 +12 2.0 +12 3.0 +120 2.0 +120 3.0 +125 2.0 +125 3.0 +126 1.0 +126 1.5 +128 3.0 +128 4.5 +129 2.0 +129 3.0 +131 1.0 +131 1.5 +133 1.0 +133 1.5 +134 2.0 +134 3.0 +136 1.0 +136 1.5 +137 2.0 +137 3.0 +138 4.0 +138 6.0 +143 1.0 +143 1.5 +145 1.0 +145 1.5 +146 2.0 +146 3.0 +149 2.0 +149 3.0 +15 2.0 +15 3.0 +150 1.0 +150 1.5 +152 2.0 +152 3.0 +153 1.0 +153 1.5 +155 1.0 +155 1.5 +156 1.0 +156 1.5 +157 1.0 +157 1.5 +158 1.0 +158 1.5 +160 1.0 +160 1.5 +162 1.0 +162 1.5 +163 1.0 +163 1.5 +164 2.0 +164 3.0 +165 2.0 +165 3.0 +166 1.0 +166 1.5 +167 3.0 +167 4.5 +168 1.0 +168 1.5 +169 4.0 +169 6.0 +17 1.0 +17 1.5 +170 1.0 +170 1.5 +172 2.0 +172 3.0 +174 2.0 +174 3.0 +175 2.0 +175 3.0 +176 2.0 +176 3.0 +177 1.0 +177 1.5 +178 1.0 +178 1.5 +179 2.0 +179 3.0 +18 2.0 +18 3.0 +180 1.0 +180 1.5 +181 1.0 +181 1.5 +183 1.0 +183 1.5 +186 1.0 +186 1.5 +187 3.0 +187 4.5 +189 1.0 +189 1.5 +19 1.0 +19 1.5 +190 1.0 +190 1.5 +191 2.0 +191 3.0 +192 1.0 +192 1.5 +193 3.0 +193 4.5 +194 1.0 +194 1.5 +195 2.0 +195 3.0 +196 1.0 +196 1.5 +197 2.0 +197 3.0 +199 3.0 +199 4.5 +2 1.0 +2 1.5 +20 1.0 +20 1.5 +200 2.0 +200 3.0 +201 1.0 +201 1.5 +202 1.0 +202 1.5 +203 2.0 +203 3.0 +205 2.0 +205 3.0 +207 2.0 +207 3.0 +208 3.0 +208 4.5 +209 2.0 +209 3.0 +213 2.0 +213 3.0 +214 1.0 +214 1.5 +216 2.0 +216 3.0 +217 2.0 +217 3.0 +218 1.0 +218 1.5 +219 2.0 +219 3.0 +221 2.0 +221 3.0 +222 1.0 +222 1.5 +223 2.0 +223 3.0 +224 2.0 +224 3.0 +226 1.0 +226 1.5 +228 1.0 +228 1.5 +229 2.0 +229 3.0 +230 5.0 +230 7.5 +233 2.0 +233 3.0 +235 1.0 +235 1.5 +237 2.0 +237 3.0 +238 2.0 +238 3.0 +239 2.0 +239 3.0 +24 2.0 +24 3.0 +241 1.0 +241 1.5 +242 2.0 +242 3.0 +244 1.0 +244 1.5 +247 1.0 +247 1.5 +248 1.0 +248 1.5 +249 1.0 +249 1.5 +252 1.0 +252 1.5 +255 2.0 +255 3.0 +256 2.0 +256 3.0 +257 1.0 +257 1.5 +258 1.0 +258 1.5 +26 2.0 +26 3.0 +260 1.0 +260 1.5 +262 1.0 +262 1.5 +263 1.0 +263 1.5 +265 2.0 +265 3.0 +266 1.0 +266 1.5 +27 1.0 +27 1.5 +272 2.0 +272 3.0 +273 3.0 +273 4.5 +274 1.0 +274 1.5 +275 1.0 +275 1.5 +277 4.0 +277 6.0 +278 2.0 +278 3.0 +28 1.0 +28 1.5 +280 2.0 +280 3.0 +281 2.0 +281 3.0 +282 2.0 +282 3.0 +283 1.0 +283 1.5 +284 1.0 +284 1.5 +285 1.0 +285 1.5 +286 1.0 +286 1.5 +287 1.0 +287 1.5 +288 2.0 +288 3.0 +289 1.0 +289 1.5 +291 1.0 +291 1.5 +292 1.0 +292 1.5 +296 1.0 +296 1.5 +298 3.0 +298 4.5 +30 1.0 +30 1.5 +302 1.0 +302 1.5 +305 1.0 +305 1.5 +306 1.0 +306 1.5 +307 2.0 +307 3.0 +308 1.0 +308 1.5 +309 2.0 +309 3.0 +310 1.0 +310 1.5 +311 3.0 +311 4.5 +315 1.0 +315 1.5 +316 3.0 +316 4.5 +317 2.0 +317 3.0 +318 3.0 +318 4.5 +321 2.0 +321 3.0 +322 2.0 +322 3.0 +323 1.0 +323 1.5 +325 2.0 +325 3.0 +327 3.0 +327 4.5 +33 1.0 +33 1.5 +331 2.0 +331 3.0 +332 1.0 +332 1.5 +333 2.0 +333 3.0 +335 1.0 +335 1.5 +336 1.0 +336 1.5 +338 1.0 +338 1.5 +339 1.0 +339 1.5 +34 1.0 +34 1.5 +341 1.0 +341 1.5 +342 2.0 +342 3.0 +344 2.0 +344 3.0 +345 1.0 +345 1.5 +348 5.0 +348 7.5 +35 3.0 +35 4.5 +351 1.0 +351 1.5 +353 2.0 +353 3.0 +356 1.0 +356 1.5 +360 1.0 +360 1.5 +362 1.0 +362 1.5 +364 1.0 +364 1.5 +365 1.0 +365 1.5 +366 1.0 +366 1.5 +367 2.0 +367 3.0 +368 1.0 +368 1.5 +369 3.0 +369 4.5 +37 2.0 +37 3.0 +373 1.0 +373 1.5 +374 1.0 +374 1.5 +375 1.0 +375 1.5 +377 1.0 +377 1.5 +378 1.0 +378 1.5 +379 1.0 +379 1.5 +382 2.0 +382 3.0 +384 3.0 +384 4.5 +386 1.0 +386 1.5 +389 1.0 +389 1.5 +392 1.0 +392 1.5 +393 1.0 +393 1.5 +394 1.0 +394 1.5 +395 2.0 +395 3.0 +396 3.0 +396 4.5 +397 2.0 +397 3.0 +399 2.0 +399 3.0 +4 1.0 +4 1.5 +400 1.0 +400 1.5 +401 5.0 +401 7.5 +402 1.0 +402 1.5 +403 3.0 +403 4.5 +404 2.0 +404 3.0 +406 4.0 +406 6.0 +407 1.0 +407 1.5 +409 3.0 +409 4.5 +41 1.0 +41 1.5 +411 1.0 +411 1.5 +413 2.0 +413 3.0 +414 2.0 +414 3.0 +417 3.0 +417 4.5 +418 1.0 +418 1.5 +419 1.0 +419 1.5 +42 2.0 +42 3.0 +421 1.0 +421 1.5 +424 2.0 +424 3.0 +427 1.0 +427 1.5 +429 2.0 +429 3.0 +43 1.0 +43 1.5 +430 3.0 +430 4.5 +431 3.0 +431 4.5 +432 1.0 +432 1.5 +435 1.0 +435 1.5 +436 1.0 +436 1.5 +437 1.0 +437 1.5 +438 3.0 +438 4.5 +439 2.0 +439 3.0 +44 1.0 +44 1.5 +443 1.0 +443 1.5 +444 1.0 +444 1.5 +446 1.0 +446 1.5 +448 1.0 +448 1.5 +449 1.0 +449 1.5 +452 1.0 +452 1.5 +453 1.0 +453 1.5 +454 3.0 +454 4.5 +455 1.0 +455 1.5 +457 1.0 +457 1.5 +458 2.0 +458 3.0 +459 2.0 +459 3.0 +460 1.0 +460 1.5 +462 2.0 +462 3.0 +463 2.0 +463 3.0 +466 3.0 +466 4.5 +467 1.0 +467 1.5 +468 4.0 +468 6.0 +469 5.0 +469 7.5 +47 1.0 +47 1.5 +470 1.0 +470 1.5 +472 1.0 +472 1.5 +475 1.0 +475 1.5 +477 1.0 +477 1.5 +478 2.0 +478 3.0 +479 1.0 +479 1.5 +480 3.0 +480 4.5 +481 1.0 +481 1.5 +482 1.0 +482 1.5 +483 1.0 +483 1.5 +484 1.0 +484 1.5 +485 1.0 +485 1.5 +487 1.0 +487 1.5 +489 4.0 +489 6.0 +490 1.0 +490 1.5 +491 1.0 +491 1.5 +492 2.0 +492 3.0 +493 1.0 +493 1.5 +494 1.0 +494 1.5 +495 1.0 +495 1.5 +496 1.0 +496 1.5 +497 1.0 +497 1.5 +498 3.0 +498 4.5 +5 3.0 +5 4.5 +51 2.0 +51 3.0 +53 1.0 +53 1.5 +54 1.0 +54 1.5 +57 1.0 +57 1.5 +58 2.0 +58 3.0 +64 1.0 +64 1.5 +65 1.0 +65 1.5 +66 1.0 +66 1.5 +67 2.0 +67 3.0 +69 1.0 +69 1.5 +70 3.0 +70 4.5 +72 2.0 +72 3.0 +74 1.0 +74 1.5 +76 2.0 +76 3.0 +77 1.0 +77 1.5 +78 1.0 +78 1.5 +8 1.0 +8 1.5 +80 1.0 +80 1.5 +82 1.0 +82 1.5 +83 2.0 +83 3.0 +84 2.0 +84 3.0 +85 1.0 +85 1.5 +86 1.0 +86 1.5 +87 1.0 +87 1.5 +9 1.0 +9 1.5 +90 3.0 +90 4.5 +92 1.0 +92 1.5 +95 2.0 +95 3.0 +96 1.0 +96 1.5 +97 2.0 +97 3.0 +98 2.0 +98 3.0 diff --git ql/src/test/results/clientpositive/union_remove_7.q.out ql/src/test/results/clientpositive/llap/union_remove_7.q.out similarity index 58% rename from ql/src/test/results/clientpositive/union_remove_7.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_7.q.out index b3014765f6..2f0c512c09 100644 --- ql/src/test/results/clientpositive/union_remove_7.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_7.q.out @@ -46,50 +46,70 @@ POSTHOOK: Input: default@inputtbl1_n17 POSTHOOK: Output: default@outputtbl1_n24 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n17 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n17 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n24 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n24 + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n24 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -101,47 +121,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n24 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n17 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n24 - PREHOOK: query: insert overwrite table outputTbl1_n24 SELECT * FROM ( @@ -182,8 +161,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 - totalSize 178 + numFiles 4 + totalSize 336 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_8.q.out ql/src/test/results/clientpositive/llap/union_remove_8.q.out similarity index 57% rename from ql/src/test/results/clientpositive/union_remove_8.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_8.q.out index 942b54985c..91d2924270 100644 --- ql/src/test/results/clientpositive/union_remove_8.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_8.q.out @@ -50,51 +50,86 @@ POSTHOOK: Input: default@inputtbl1_n9 POSTHOOK: Output: default@outputtbl1_n12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage - Stage-3 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n9 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n9 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Select Operator + expressions: key (type: string), 1L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n12 + Select Operator + expressions: key (type: string), 2L (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n12 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n12 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n12 + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -106,46 +141,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n12 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n9 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n12 - Execution mode: vectorized - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n9 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2L (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n12 - Execution mode: vectorized - PREHOOK: query: insert overwrite table outputTbl1_n12 SELECT * FROM ( @@ -190,8 +185,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 3 - totalSize 271 + numFiles 4 + totalSize 350 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/union_remove_9.q.out ql/src/test/results/clientpositive/llap/union_remove_9.q.out similarity index 52% rename from ql/src/test/results/clientpositive/union_remove_9.q.out rename to ql/src/test/results/clientpositive/llap/union_remove_9.q.out index 2e2693be37..c9b1d55beb 100644 --- ql/src/test/results/clientpositive/union_remove_9.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_9.q.out @@ -54,64 +54,112 @@ POSTHOOK: Input: default@inputtbl1_n15 POSTHOOK: Output: default@outputtbl1_n22 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 - Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n15 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 5 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: inputtbl1_n15 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: inputtbl1_n15 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n22 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: inputtbl1_n15 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n22 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n22 - - Stage: Stage-6 - Conditional Operator + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n22 + Union 4 + Vertex: Union 4 - Stage: Stage-3 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -123,72 +171,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1_n22 - Stage: Stage-2 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-4 - Merge File Operator - Map Operator Tree: - RCFile Merge Operator - merge level: block - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n15 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n22 - TableScan - alias: inputtbl1_n15 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), 2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n22 - PREHOOK: query: insert overwrite table outputTbl1_n22 SELECT * FROM ( @@ -237,8 +219,8 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 - numFiles 2 - totalSize 192 + numFiles 4 + totalSize 350 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/llap/union_remove_plan.q.out ql/src/test/results/clientpositive/llap/union_remove_plan.q.out new file mode 100644 index 0000000000..070c24b1ed --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_remove_plan.q.out @@ -0,0 +1,120 @@ +PREHOOK: query: create table if not exists test_table(column1 string, column2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table +POSTHOOK: query: create table if not exists test_table(column1 string, column2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table +PREHOOK: query: insert into test_table values('a',1),('b',2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_table +POSTHOOK: query: insert into test_table values('a',1),('b',2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_table +POSTHOOK: Lineage: test_table.column1 SCRIPT [] +POSTHOOK: Lineage: test_table.column2 SCRIPT [] +PREHOOK: query: explain +select column1 from test_table group by column1 +union all +select column1 from test_table group by column1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table +#### A masked pattern was here #### +POSTHOOK: query: explain +select column1 from test_table group by column1 +union all +select column1 from test_table group by column1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_table + Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: column1 (type: string) + outputColumnNames: column1 + Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: column1 (type: string) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select column1 from test_table group by column1 +union all +select column1 from test_table group by column1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table +#### A masked pattern was here #### +POSTHOOK: query: select column1 from test_table group by column1 +union all +select column1 from test_table group by column1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table +#### A masked pattern was here #### +a +b +a +b +PREHOOK: query: drop table test_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test_table +PREHOOK: Output: default@test_table +POSTHOOK: query: drop table test_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test_table +POSTHOOK: Output: default@test_table diff --git ql/src/test/results/clientpositive/llap/union_view.q.out ql/src/test/results/clientpositive/llap/union_view.q.out new file mode 100644 index 0000000000..929d76458b --- /dev/null +++ ql/src/test/results/clientpositive/llap/union_view.q.out @@ -0,0 +1,1092 @@ +PREHOOK: query: CREATE TABLE src_union_1_n0 (key int, value string) PARTITIONED BY (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_union_1_n0 +POSTHOOK: query: CREATE TABLE src_union_1_n0 (key int, value string) PARTITIONED BY (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_union_1_n0 +PREHOOK: query: CREATE TABLE src_union_2_n0 (key int, value string) PARTITIONED BY (ds string, part_1 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_union_2_n0 +POSTHOOK: query: CREATE TABLE src_union_2_n0 (key int, value string) PARTITIONED BY (ds string, part_1 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_union_2_n0 +PREHOOK: query: CREATE TABLE src_union_3_n0(key int, value string) PARTITIONED BY (ds string, part_1 string, part_2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_union_3_n0 +POSTHOOK: query: CREATE TABLE src_union_3_n0(key int, value string) PARTITIONED BY (ds string, part_1 string, part_2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_union_3_n0 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and (ds = '1')) (type: boolean) + Filter Operator + predicate: (key = 86) (type: boolean) + Select Operator + expressions: 86 (type: int), value (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and (ds = '2')) (type: boolean) + Filter Operator + predicate: (key = 86) (type: boolean) + Select Operator + expressions: 86 (type: int), value (type: string), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and (ds = '3')) (type: boolean) + Filter Operator + predicate: (key = 86) (type: boolean) + Select Operator + expressions: 86 (type: int), value (type: string), '3' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +86 val_86 1 +86 val_86 2 +86 val_86 2 +86 val_86 3 +86 val_86 3 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +500 +1000 +1000 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '1')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '1' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 3 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '2')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '2' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and (ds = '3')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '3')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '3' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 3 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and (ds = '3')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '3')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '3' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and (ds = '3')) (type: boolean) + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '3' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and ds is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 2 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), ds (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 550 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and ds is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), ds (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 825 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and ds is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), ds (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 825 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +86 val_86 1 +86 val_86 2 +86 val_86 2 +86 val_86 3 +86 val_86 3 +86 val_86 1 +86 val_86 2 +86 val_86 2 +86 val_86 3 +86 val_86 3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: (ds = '1') (type: boolean) + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: (ds = '1') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '1') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 5 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: (ds = '1') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '1') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: (ds = '2') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '2') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: (ds = '2') (type: boolean) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: (ds = '2') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '2') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: (ds = '3') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '3') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: (ds = '3') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '3') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 5 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: (ds = '3') (type: boolean) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +500 +1000 +1000 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: ((key = 86) and (ds = '4')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '4')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 3 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: ((key = 86) and (ds = '4')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key = 86) and (ds = '4')) (type: boolean) + Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: ((key = 86) and (ds = '4')) (type: boolean) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 86 (type: int), _col0 (type: string), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +86 val_86 4 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_union_1_n0 + filterExpr: (ds = '4') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '4') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 4 + Map Operator Tree: + TableScan + alias: src_union_2_n0 + filterExpr: (ds = '4') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ds = '4') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map 5 + Map Operator Tree: + TableScan + alias: src_union_3_n0 + filterExpr: (ds = '4') (type: boolean) + Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +500 diff --git ql/src/test/results/clientpositive/unionall_lateralview.q.out ql/src/test/results/clientpositive/llap/unionall_lateralview.q.out similarity index 97% rename from ql/src/test/results/clientpositive/unionall_lateralview.q.out rename to ql/src/test/results/clientpositive/llap/unionall_lateralview.q.out index db647774fe..9b8becdc96 100644 --- ql/src/test/results/clientpositive/unionall_lateralview.q.out +++ ql/src/test/results/clientpositive/llap/unionall_lateralview.q.out @@ -31,7 +31,7 @@ POSTHOOK: query: CREATE TABLE unionall_lateralview2(col1 INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@unionall_lateralview2 -Warning: Shuffle Join JOIN[18][tables = [x1, expdobj]] in Stage 'Stage-9:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [x1, expdobj]] in Stage 'Reducer 5' is a cross product PREHOOK: query: INSERT INTO unionall_lateralview2 SELECT 1 AS `col1` FROM unionall_lateralview1 diff --git ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out ql/src/test/results/clientpositive/llap/unionall_unbalancedppd.q.out similarity index 79% rename from ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out rename to ql/src/test/results/clientpositive/llap/unionall_unbalancedppd.q.out index 15415dbff8..d4bedf4d62 100644 --- ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out +++ ql/src/test/results/clientpositive/llap/unionall_unbalancedppd.q.out @@ -77,38 +77,22 @@ POSTHOOK: Input: default@union_all_bug_test_1 POSTHOOK: Input: default@union_all_bug_test_2 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: union_all_bug_test_1 - filterExpr: (if(true, f1, f2) = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (if(true, f1, f2) = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: union_all_bug_test_1 + filterExpr: (if(true, f1, f2) = 1) (type: boolean) + Filter Operator + predicate: (if(true, f1, f2) = 1) (type: boolean) + Select Operator + expressions: f1 (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: SELECT f1 FROM ( @@ -366,52 +350,63 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: union_all_bug_test_1 - filterExpr: (f1 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (f1 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Union + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: union_all_bug_test_1 + filterExpr: (f1 = 1) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: union_all_bug_test_2 - filterExpr: (f1 = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (f1 = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Union + Filter Operator + predicate: (f1 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: union_all_bug_test_2 + filterExpr: (f1 = 1) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (f1 = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/updateBasicStats.q.out ql/src/test/results/clientpositive/llap/updateBasicStats.q.out similarity index 87% rename from ql/src/test/results/clientpositive/updateBasicStats.q.out rename to ql/src/test/results/clientpositive/llap/updateBasicStats.q.out index 162114b575..b3acbcbddb 100644 --- ql/src/test/results/clientpositive/updateBasicStats.q.out +++ ql/src/test/results/clientpositive/llap/updateBasicStats.q.out @@ -28,11 +28,9 @@ STAGE PLANS: Processor Tree: TableScan alias: s_n5 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: alter table s_n5 update statistics set('numRows'='12') @@ -61,11 +59,9 @@ STAGE PLANS: Processor Tree: TableScan alias: s_n5 - Statistics: Num rows: 12 Data size: 2136 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 2136 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: analyze table s_n5 compute statistics @@ -94,11 +90,9 @@ STAGE PLANS: Processor Tree: TableScan alias: s_n5 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: alter table s_n5 update statistics set('numRows'='1212', 'rawDataSize'='500500') @@ -127,11 +121,9 @@ STAGE PLANS: Processor Tree: TableScan alias: s_n5 - Statistics: Num rows: 1212 Data size: 215736 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1212 Data size: 215736 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: CREATE TABLE calendarp_n0 (`year` int) partitioned by (p int) @@ -172,11 +164,9 @@ STAGE PLANS: TableScan alias: calendarp_n0 filterExpr: (p = 1) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: year (type: int), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: alter table calendarp_n0 partition (p=1) update statistics set('numRows'='1000020000', 'rawDataSize'='300040000') @@ -209,11 +199,9 @@ STAGE PLANS: TableScan alias: calendarp_n0 filterExpr: (p = 1) (type: boolean) - Statistics: Num rows: 1000020000 Data size: 4000080000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: year (type: int), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000020000 Data size: 8000160000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: create table src_stat_part_two_n0(key string, value string) partitioned by (px int, py string) @@ -281,11 +269,9 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: ((px = 1) and (py = 'a')) (type: boolean) - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), 'a' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 267 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select * from src_stat_part_two_n0 where px=1 @@ -311,11 +297,9 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: (px = 1) (type: boolean) - Statistics: Num rows: 11 Data size: 3982 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), py (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 4026 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: alter table src_stat_part_two_n0 partition (px=1, py='a') update statistics set('numRows'='1000020000', 'rawDataSize'='300040000') @@ -348,11 +332,9 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: ((px = 1) and (py = 'a')) (type: boolean) - Statistics: Num rows: 1000020000 Data size: 178003560000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), 'a' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000020000 Data size: 267005340000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select * from src_stat_part_two_n0 where px=1 @@ -378,11 +360,9 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: (px = 1) (type: boolean) - Statistics: Num rows: 1000020010 Data size: 362007243620 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), py (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000020010 Data size: 366007323660 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: alter table src_stat_part_two_n0 partition (px=1) update statistics set('numRows'='1000020000', 'rawDataSize'='300040000') @@ -418,11 +398,9 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: ((px = 1) and (py = 'a')) (type: boolean) - Statistics: Num rows: 1000020000 Data size: 178003560000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), 'a' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000020000 Data size: 267005340000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select * from src_stat_part_two_n0 where px=1 @@ -448,10 +426,8 @@ STAGE PLANS: TableScan alias: src_stat_part_two_n0 filterExpr: (px = 1) (type: boolean) - Statistics: Num rows: 2000040000 Data size: 724014480000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string), 1 (type: int), py (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000040000 Data size: 732014640000 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git ql/src/test/results/clientpositive/varchar_serde.q.out ql/src/test/results/clientpositive/llap/varchar_serde.q.out similarity index 90% rename from ql/src/test/results/clientpositive/varchar_serde.q.out rename to ql/src/test/results/clientpositive/llap/varchar_serde.q.out index 62797e028a..abc84efce7 100644 --- ql/src/test/results/clientpositive/varchar_serde.q.out +++ ql/src/test/results/clientpositive/llap/varchar_serde.q.out @@ -54,24 +54,24 @@ POSTHOOK: query: load data local inpath '../../data/files/srcbucket0.txt' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@varchar_serde_regex -PREHOOK: query: select * from varchar_serde_regex limit 5 +PREHOOK: query: select * from varchar_serde_regex order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_regex #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_regex limit 5 +POSTHOOK: query: select * from varchar_serde_regex order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_regex #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_regex group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_regex group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_regex #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_regex group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_regex group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_regex #### A masked pattern was here #### @@ -114,24 +114,24 @@ POSTHOOK: Input: default@varchar_serde_regex POSTHOOK: Output: default@varchar_serde_lb POSTHOOK: Lineage: varchar_serde_lb.key SIMPLE [(varchar_serde_regex)varchar_serde_regex.FieldSchema(name:key, type:varchar(10), comment:), ] POSTHOOK: Lineage: varchar_serde_lb.value SIMPLE [(varchar_serde_regex)varchar_serde_regex.FieldSchema(name:value, type:varchar(20), comment:), ] -PREHOOK: query: select * from varchar_serde_lb limit 5 +PREHOOK: query: select * from varchar_serde_lb order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_lb #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_lb limit 5 +POSTHOOK: query: select * from varchar_serde_lb order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_lb #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_lb group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_lb group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_lb #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_lb group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_lb group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_lb #### A masked pattern was here #### @@ -174,24 +174,24 @@ POSTHOOK: Input: default@varchar_serde_lb POSTHOOK: Output: default@varchar_serde_ls POSTHOOK: Lineage: varchar_serde_ls.key SIMPLE [(varchar_serde_lb)varchar_serde_lb.FieldSchema(name:key, type:varchar(10), comment:null), ] POSTHOOK: Lineage: varchar_serde_ls.value SIMPLE [(varchar_serde_lb)varchar_serde_lb.FieldSchema(name:value, type:varchar(20), comment:null), ] -PREHOOK: query: select * from varchar_serde_ls limit 5 +PREHOOK: query: select * from varchar_serde_ls order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_ls #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_ls limit 5 +POSTHOOK: query: select * from varchar_serde_ls order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_ls #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_ls group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_ls group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_ls #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_ls group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_ls group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_ls #### A masked pattern was here #### @@ -234,24 +234,24 @@ POSTHOOK: Input: default@varchar_serde_ls POSTHOOK: Output: default@varchar_serde_c POSTHOOK: Lineage: varchar_serde_c.key SIMPLE [(varchar_serde_ls)varchar_serde_ls.FieldSchema(name:key, type:varchar(10), comment:null), ] POSTHOOK: Lineage: varchar_serde_c.value SIMPLE [(varchar_serde_ls)varchar_serde_ls.FieldSchema(name:value, type:varchar(20), comment:null), ] -PREHOOK: query: select * from varchar_serde_c limit 5 +PREHOOK: query: select * from varchar_serde_c order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_c #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_c limit 5 +POSTHOOK: query: select * from varchar_serde_c order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_c #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_c group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_c group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_c #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_c group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_c group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_c #### A masked pattern was here #### @@ -294,24 +294,24 @@ POSTHOOK: Input: default@varchar_serde_c POSTHOOK: Output: default@varchar_serde_lbc POSTHOOK: Lineage: varchar_serde_lbc.key SIMPLE [(varchar_serde_c)varchar_serde_c.FieldSchema(name:key, type:varchar(10), comment:null), ] POSTHOOK: Lineage: varchar_serde_lbc.value SIMPLE [(varchar_serde_c)varchar_serde_c.FieldSchema(name:value, type:varchar(20), comment:null), ] -PREHOOK: query: select * from varchar_serde_lbc limit 5 +PREHOOK: query: select * from varchar_serde_lbc order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_lbc #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_lbc limit 5 +POSTHOOK: query: select * from varchar_serde_lbc order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_lbc #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_lbc group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_lbc group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_lbc #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_lbc group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_lbc group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_lbc #### A masked pattern was here #### @@ -354,24 +354,24 @@ POSTHOOK: Input: default@varchar_serde_lbc POSTHOOK: Output: default@varchar_serde_orc POSTHOOK: Lineage: varchar_serde_orc.key SIMPLE [(varchar_serde_lbc)varchar_serde_lbc.FieldSchema(name:key, type:varchar(10), comment:null), ] POSTHOOK: Lineage: varchar_serde_orc.value SIMPLE [(varchar_serde_lbc)varchar_serde_lbc.FieldSchema(name:value, type:varchar(20), comment:null), ] -PREHOOK: query: select * from varchar_serde_orc limit 5 +PREHOOK: query: select * from varchar_serde_orc order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_orc #### A masked pattern was here #### -POSTHOOK: query: select * from varchar_serde_orc limit 5 +POSTHOOK: query: select * from varchar_serde_orc order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_orc #### A masked pattern was here #### -474 val_475 -62 val_63 -468 val_469 -272 val_273 -448 val_449 -PREHOOK: query: select value, count(*) from varchar_serde_orc group by value limit 5 +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +PREHOOK: query: select value, count(*) from varchar_serde_orc group by value order by value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@varchar_serde_orc #### A masked pattern was here #### -POSTHOOK: query: select value, count(*) from varchar_serde_orc group by value limit 5 +POSTHOOK: query: select value, count(*) from varchar_serde_orc group by value order by value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_serde_orc #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/vector_const.q.out ql/src/test/results/clientpositive/llap/vector_const.q.out similarity index 61% rename from ql/src/test/results/clientpositive/vector_const.q.out rename to ql/src/test/results/clientpositive/llap/vector_const.q.out index 25870ae430..fc914a0a3e 100644 --- ql/src/test/results/clientpositive/vector_const.q.out +++ ql/src/test/results/clientpositive/llap/vector_const.q.out @@ -31,23 +31,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: varchar_const_1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'FF' (type: varchar(3)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: varchar_const_1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'FF' (type: varchar(3)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_col_scalar_division.q.out ql/src/test/results/clientpositive/llap/vector_decimal_col_scalar_division.q.out new file mode 100644 index 0000000000..f1398fbb30 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_decimal_col_scalar_division.q.out @@ -0,0 +1,166 @@ +PREHOOK: query: CREATE temporary TABLE `catalog_Sales`( + `cs_quantity` int, + `cs_wholesale_cost` decimal(7,2), + `cs_list_price` decimal(7,2), + `cs_sales_price` decimal(7,2), + `cs_ext_discount_amt` decimal(7,2), + `cs_ext_sales_price` decimal(7,2), + `cs_ext_wholesale_cost` decimal(7,2), + `cs_ext_list_price` decimal(7,2), + `cs_ext_tax` decimal(7,2), + `cs_coupon_amt` decimal(7,2), + `cs_ext_ship_cost` decimal(7,2), + `cs_net_paid` decimal(7,2), + `cs_net_paid_inc_tax` decimal(7,2), + `cs_net_paid_inc_ship` decimal(7,2), + `cs_net_paid_inc_ship_tax` decimal(7,2), + `cs_net_profit` decimal(7,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@catalog_Sales +POSTHOOK: query: CREATE temporary TABLE `catalog_Sales`( + `cs_quantity` int, + `cs_wholesale_cost` decimal(7,2), + `cs_list_price` decimal(7,2), + `cs_sales_price` decimal(7,2), + `cs_ext_discount_amt` decimal(7,2), + `cs_ext_sales_price` decimal(7,2), + `cs_ext_wholesale_cost` decimal(7,2), + `cs_ext_list_price` decimal(7,2), + `cs_ext_tax` decimal(7,2), + `cs_coupon_amt` decimal(7,2), + `cs_ext_ship_cost` decimal(7,2), + `cs_net_paid` decimal(7,2), + `cs_net_paid_inc_tax` decimal(7,2), + `cs_net_paid_inc_ship` decimal(7,2), + `cs_net_paid_inc_ship_tax` decimal(7,2), + `cs_net_profit` decimal(7,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@catalog_Sales +PREHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales +PREHOOK: type: QUERY +PREHOOK: Input: default@catalog_sales +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales +POSTHOOK: type: QUERY +POSTHOOK: Input: default@catalog_sales +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: catalog_sales + Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:cs_quantity:int, 1:cs_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:cs_list_price:decimal(7,2)/DECIMAL_64, 3:cs_sales_price:decimal(7,2)/DECIMAL_64, 4:cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, 5:cs_ext_sales_price:decimal(7,2)/DECIMAL_64, 6:cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 7:cs_ext_list_price:decimal(7,2)/DECIMAL_64, 8:cs_ext_tax:decimal(7,2)/DECIMAL_64, 9:cs_coupon_amt:decimal(7,2)/DECIMAL_64, 10:cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, 11:cs_net_paid:decimal(7,2)/DECIMAL_64, 12:cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, 13:cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, 14:cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, 15:cs_net_profit:decimal(7,2)/DECIMAL_64, 16:ROW__ID:struct] + Select Operator + expressions: ((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) (type: decimal(14,6)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [20] + selectExpressions: Decimal64ColDivideDecimal64Scalar(col 19:decimal(10,2)/DECIMAL_64, decimal64Val 200, decimalVal 2)(children: Decimal64ColAddDecimal64Column(col 18:decimal(9,2)/DECIMAL_64, col 5:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 17:decimal(8,2)/DECIMAL_64, col 4:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 7:decimal(7,2)/DECIMAL_64, col 6:decimal(7,2)/DECIMAL_64) -> 17:decimal(8,2)/DECIMAL_64) -> 18:decimal(9,2)/DECIMAL_64) -> 19:decimal(10,2)/DECIMAL_64) -> 20:decimal(14,6)/DECIMAL_64 + Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(_col0) + Group By Vectorization: + aggregators: VectorUDAFMaxDecimal64(col 20:decimal(14,6)/DECIMAL_64) -> decimal(14,6)/DECIMAL_64 + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(14,6) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(14,6)) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 16 + includeColumns: [4, 5, 6, 7] + dataColumns: cs_quantity:int, cs_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_list_price:decimal(7,2)/DECIMAL_64, cs_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, cs_ext_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_ext_list_price:decimal(7,2)/DECIMAL_64, cs_ext_tax:decimal(7,2)/DECIMAL_64, cs_coupon_amt:decimal(7,2)/DECIMAL_64, cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, cs_net_paid:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, cs_net_profit:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, decimal(14,6)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(14,6)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFMaxDecimal64(col 0:decimal(14,6)/DECIMAL_64) -> decimal(14,6)/DECIMAL_64 + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/vector_decimal_partition.q.out ql/src/test/results/clientpositive/llap/vector_decimal_partition.q.out new file mode 100644 index 0000000000..4ca6d83907 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_decimal_partition.q.out @@ -0,0 +1,178 @@ +PREHOOK: query: DROP TABLE IF EXISTS decimal_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS decimal_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE decimal_part (id DECIMAL(4,0), foo VARCHAR(10)) + PARTITIONED BY (nr_bank DECIMAL(4,0)) +STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_part +POSTHOOK: query: CREATE TABLE decimal_part (id DECIMAL(4,0), foo VARCHAR(10)) + PARTITIONED BY (nr_bank DECIMAL(4,0)) +STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_part +PREHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 88) VALUES (1, 'test') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@decimal_part@nr_bank=88 +POSTHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 88) VALUES (1, 'test') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@decimal_part@nr_bank=88 +POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=88).foo SCRIPT [] +POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=88).id SCRIPT [] +PREHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 8801) VALUES (1, '8801') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@decimal_part@nr_bank=8801 +POSTHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 8801) VALUES (1, '8801') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@decimal_part@nr_bank=8801 +POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=8801).foo SCRIPT [] +POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=8801).id SCRIPT [] +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_part +PREHOOK: Input: default@decimal_part@nr_bank=88 +PREHOOK: Input: default@decimal_part@nr_bank=8801 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_part +POSTHOOK: Input: default@decimal_part@nr_bank=88 +POSTHOOK: Input: default@decimal_part@nr_bank=8801 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: decimal_part + Statistics: Num rows: 2 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: nr_bank (type: decimal(4,0)) + outputColumnNames: nr_bank + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + Statistics: Num rows: 2 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + Group By Vectorization: + aggregators: VectorUDAFCountStar(*) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: ConvertDecimal64ToDecimal(col 2:decimal(4,0)/DECIMAL_64) -> 4:decimal(4,0) + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + keys: nr_bank (type: decimal(4,0)) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: decimal(4,0)) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: decimal(4,0)) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:decimal(4,0) + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [0] + keys: KEY._col0 (type: decimal(4,0)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: bigint), _col0 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 0] + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_part +PREHOOK: Input: default@decimal_part@nr_bank=88 +PREHOOK: Input: default@decimal_part@nr_bank=8801 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_part +POSTHOOK: Input: default@decimal_part@nr_bank=88 +POSTHOOK: Input: default@decimal_part@nr_bank=8801 +#### A masked pattern was here #### +1 88 +1 8801 diff --git ql/src/test/results/clientpositive/llap/vector_delete_orig_table.q.out ql/src/test/results/clientpositive/llap/vector_delete_orig_table.q.out new file mode 100644 index 0000000000..331cd75465 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_delete_orig_table.q.out @@ -0,0 +1,195 @@ +PREHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dot +POSTHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dot +PREHOOK: query: explain vectorization detail +select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: acid_dot + Statistics: Num rows: 5865 Data size: 2956160 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [] + Statistics: Num rows: 5865 Data size: 2956160 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + Group By Vectorization: + aggregators: VectorUDAFCountStar(*) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:bigint + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:bigint + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12288 +PREHOOK: query: select count(*) from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +8 +PREHOOK: query: delete from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +PREHOOK: Output: default@acid_dot +POSTHOOK: query: delete from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +POSTHOOK: Output: default@acid_dot +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12280 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/vector_empty_where.q.out ql/src/test/results/clientpositive/llap/vector_empty_where.q.out new file mode 100644 index 0000000000..7f9477c29c --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_empty_where.q.out @@ -0,0 +1,684 @@ +PREHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cstring1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cstring1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 899146 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastStringToBoolean(col 6) -> 13:boolean) + predicate: cstring1 (type: string) + Statistics: Num rows: 6144 Data size: 449620 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + Statistics: Num rows: 6144 Data size: 449620 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] + keys: cint (type: int) + minReductionHashAggr: 0.49983722 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + Group By Vectorization: + aggregators: VectorUDAFCount(col 0:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count (distinct cint) from alltypesorc where cstring1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select count (distinct cint) from alltypesorc where cstring1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +6041 +PREHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastLongToBooleanViaLongToLong(col 2:int) -> 13:boolean) + predicate: cint (type: int) + Statistics: Num rows: 6144 Data size: 18348 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] + keys: cint (type: int) + minReductionHashAggr: 0.5030924 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + Group By Vectorization: + aggregators: VectorUDAFCount(col 0:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count (distinct cint) from alltypesorc where cint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select count (distinct cint) from alltypesorc where cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +6082 +PREHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cfloat +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where cfloat +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 73392 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastDoubleToBooleanViaDoubleToLong(col 4:float) -> 13:boolean) + predicate: cfloat (type: float) + Statistics: Num rows: 6144 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + Statistics: Num rows: 6144 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] + keys: cint (type: int) + minReductionHashAggr: 0.49983722 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + Group By Vectorization: + aggregators: VectorUDAFCount(col 0:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count (distinct cint) from alltypesorc where cfloat +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select count (distinct cint) from alltypesorc where cfloat +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +3022 +PREHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where ctimestamp1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select count (distinct cint) from alltypesorc where ctimestamp1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 403656 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastTimestampToBoolean(col 8:timestamp) -> 13:boolean) + predicate: ctimestamp1 (type: timestamp) + Statistics: Num rows: 6144 Data size: 201828 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + Statistics: Num rows: 6144 Data size: 201828 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] + keys: cint (type: int) + minReductionHashAggr: 0.49983722 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:int + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [] + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + Group By Vectorization: + aggregators: VectorUDAFCount(col 0:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count (distinct cint) from alltypesorc where ctimestamp1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select count (distinct cint) from alltypesorc where ctimestamp1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +3022 diff --git ql/src/test/results/clientpositive/vector_gather_stats.q.out ql/src/test/results/clientpositive/llap/vector_gather_stats.q.out similarity index 86% rename from ql/src/test/results/clientpositive/vector_gather_stats.q.out rename to ql/src/test/results/clientpositive/llap/vector_gather_stats.q.out index fd17a20f3a..3f0c49788e 100644 --- ql/src/test/results/clientpositive/vector_gather_stats.q.out +++ ql/src/test/results/clientpositive/llap/vector_gather_stats.q.out @@ -77,23 +77,27 @@ PLAN VECTORIZATION: STAGE DEPENDENCIES: Stage-0 is a root stage - Stage-1 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-0 - Map Reduce - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: TABLESCAN operator: gather stats not supported - vectorized: false + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cd + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: TABLESCAN operator: gather stats not supported + vectorized: false - Stage: Stage-1 + Stage: Stage-2 Stats Work Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/vector_non_constant_in_expr.q.out ql/src/test/results/clientpositive/llap/vector_non_constant_in_expr.q.out new file mode 100644 index 0000000000..8b58bd3129 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_non_constant_in_expr.q.out @@ -0,0 +1,59 @@ +PREHOOK: query: explain vectorization SELECT * FROM alltypesorc WHERE cint in (ctinyint, cbigint) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization SELECT * FROM alltypesorc WHERE cint in (ctinyint, cbigint) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + filterExpr: ((cint = UDFToInteger(ctinyint)) or (UDFToLong(cint) = cbigint)) (type: boolean) + Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((cint = UDFToInteger(ctinyint)) or (UDFToLong(cint) = cbigint)) (type: boolean) + Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/vector_outer_join_no_keys.q.out ql/src/test/results/clientpositive/llap/vector_outer_join_no_keys.q.out new file mode 100644 index 0000000000..2aa6ce1cfa --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_outer_join_no_keys.q.out @@ -0,0 +1,430 @@ +PREHOOK: query: create temporary table foo(x int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@foo +POSTHOOK: query: create temporary table foo(x int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@foo +PREHOOK: query: insert into foo values(1),(2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@foo +POSTHOOK: query: insert into foo values(1),(2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@foo +POSTHOOK: Lineage: foo.x SCRIPT [] +PREHOOK: query: create temporary table bar(y int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@bar +POSTHOOK: query: create temporary table bar(y int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@bar +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: explain vectorization detail +select count(*) from bar right outer join foo +PREHOOK: type: QUERY +PREHOOK: Input: default@bar +PREHOOK: Input: default@foo +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select count(*) from bar right outer join foo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bar +POSTHOOK: Input: default@foo +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: bar + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:y:int, 1:ROW__ID:struct] + Select Operator + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [] + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: y:int + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Map 2 + Map Operator Tree: + TableScan + alias: foo + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:x:int, 1:ROW__ID:struct] + Select Operator + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [] + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + filter predicates: + 0 + 1 {true} + keys: + 0 + 1 + Map Join Vectorization: + className: VectorMapJoinOuterFilteredOperator + native: false + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nativeConditionsNotMet: Outer Join has keys IS false + input vertices: + 0 Map 1 + Statistics: Num rows: 2 Data size: 10 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count() + Group By Vectorization: + aggregators: VectorUDAFCountStar(*) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:bigint + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: x:int + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:bigint + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: -- = 2 + +select count(*) from bar right outer join foo +PREHOOK: type: QUERY +PREHOOK: Input: default@bar +PREHOOK: Input: default@foo +#### A masked pattern was here #### +POSTHOOK: query: -- = 2 + +select count(*) from bar right outer join foo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bar +POSTHOOK: Input: default@foo +#### A masked pattern was here #### +2 +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: -- = 2 + +explain vectorization detail +select count(*) from bar, foo +PREHOOK: type: QUERY +PREHOOK: Input: default@bar +PREHOOK: Input: default@foo +#### A masked pattern was here #### +POSTHOOK: query: -- = 2 + +explain vectorization detail +select count(*) from bar, foo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bar +POSTHOOK: Input: default@foo +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: bar + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:y:int, 1:ROW__ID:struct] + Select Operator + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [] + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: y:int + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Map 2 + Map Operator Tree: + TableScan + alias: foo + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:x:int, 1:ROW__ID:struct] + Select Operator + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [] + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + Map Join Vectorization: + bigTableRetainColumnNums: [] + className: VectorMapJoinInnerBigOnlyMultiKeyOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nonOuterSmallTableKeyMapping: [] + hashTableImplementationType: OPTIMIZED + input vertices: + 0 Map 1 + Statistics: Num rows: 2 Data size: 10 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count() + Group By Vectorization: + aggregators: VectorUDAFCountStar(*) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:bigint + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: x:int + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:bigint + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: -- = 0 + +select count(*) from bar, foo +PREHOOK: type: QUERY +PREHOOK: Input: default@bar +PREHOOK: Input: default@foo +#### A masked pattern was here #### +POSTHOOK: query: -- = 0 + +select count(*) from bar, foo +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bar +POSTHOOK: Input: default@foo +#### A masked pattern was here #### +0 diff --git ql/src/test/results/clientpositive/llap/vector_tablesample_rows.q.out ql/src/test/results/clientpositive/llap/vector_tablesample_rows.q.out new file mode 100644 index 0000000000..72a9087db3 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_tablesample_rows.q.out @@ -0,0 +1,423 @@ +PREHOOK: query: explain vectorization detail +select 'key1', 'value1' from alltypesorc tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select 'key1', 'value1' from alltypesorc tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Row Limit Per Split: 1 + Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + expressions: 'key1' (type: string), 'value1' (type: string) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13, 14] + selectExpressions: ConstantVectorExpression(val key1) -> 13:string, ConstantVectorExpression(val value1) -> 14:string + Statistics: Num rows: 12288 Data size: 2187264 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 12288 Data size: 2187264 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [string, string] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select 'key1', 'value1' from alltypesorc tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select 'key1', 'value1' from alltypesorc tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +_c0 _c1 +key1 value1 +PREHOOK: query: create table decimal_2_n0 (t decimal(18,9)) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_2_n0 +POSTHOOK: query: create table decimal_2_n0 (t decimal(18,9)) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_2_n0 +PREHOOK: query: explain vectorization detail +insert overwrite table decimal_2_n0 + select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@decimal_2_n0 +POSTHOOK: query: explain vectorization detail +insert overwrite table decimal_2_n0 + select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@decimal_2_n0 +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Row Limit Per Split: 1 + Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + expressions: 17.29 (type: decimal(18,9)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13] + selectExpressions: ConstantVectorExpression(val 17.29) -> 13:decimal(18,9) + Statistics: Num rows: 12288 Data size: 1376256 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 12288 Data size: 1376256 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.decimal_2_n0 + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(18,9)] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.decimal_2_n0 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table decimal_2_n0 + select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@decimal_2_n0 +POSTHOOK: query: insert overwrite table decimal_2_n0 + select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@decimal_2_n0 +POSTHOOK: Lineage: decimal_2_n0.t EXPRESSION [] +_col0 +PREHOOK: query: select count(*) from decimal_2_n0 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2_n0 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from decimal_2_n0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2_n0 +#### A masked pattern was here #### +_c0 +1 +PREHOOK: query: drop table decimal_2_n0 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_2_n0 +PREHOOK: Output: default@decimal_2_n0 +POSTHOOK: query: drop table decimal_2_n0 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_2_n0 +POSTHOOK: Output: default@decimal_2_n0 +PREHOOK: query: explain vectorization detail +select count(1) from (select * from (Select 1 a) x order by x.a) y +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select count(1) from (select * from (Select 1 a) x order by x.a) y +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: false + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + enabledConditionsNotMet: Could not enable vectorization due to partition column names size 1 is greater than the number of table column names size 0 IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.NullRowsInputFormat + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:bigint + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) from (select * from (Select 1 a) x order by x.a) y +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from (select * from (Select 1 a) x order by x.a) y +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +_c0 +1 +PREHOOK: query: explain vectorization detail +create temporary table dual as select 1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@dual +POSTHOOK: query: explain vectorization detail +create temporary table dual as select 1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dual +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dual + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: false + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + enabledConditionsNotMet: Could not enable vectorization due to partition column names size 1 is greater than the number of table column names size 0 IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.NullRowsInputFormat + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create Table + columns: _c0 int + name: default.dual + input format: org.apache.hadoop.mapred.TextInputFormat +#### A masked pattern was here #### + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + isTemporary: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: create temporary table dual as select 1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@dual +POSTHOOK: query: create temporary table dual as select 1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dual +_c0 +PREHOOK: query: select * from dual +PREHOOK: type: QUERY +PREHOOK: Input: default@dual +#### A masked pattern was here #### +POSTHOOK: query: select * from dual +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dual +#### A masked pattern was here #### +dual._c0 +1 diff --git ql/src/test/results/clientpositive/llap/vector_windowing_row_number.q.out ql/src/test/results/clientpositive/llap/vector_windowing_row_number.q.out new file mode 100644 index 0000000000..bb15a7309f --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_windowing_row_number.q.out @@ -0,0 +1,831 @@ +PREHOOK: query: drop table row_number_test +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table row_number_test +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table row_number_test as select explode(split(repeat("w,", 2400), ",")) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@row_number_test +POSTHOOK: query: create table row_number_test as select explode(split(repeat("w,", 2400), ",")) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@row_number_test +POSTHOOK: Lineage: row_number_test.col SCRIPT [] +col +PREHOOK: query: insert into row_number_test select explode(split(repeat("x,", 1200), ",")) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@row_number_test +POSTHOOK: query: insert into row_number_test select explode(split(repeat("x,", 1200), ",")) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@row_number_test +POSTHOOK: Lineage: row_number_test.col SCRIPT [] +col +PREHOOK: query: insert into row_number_test select explode(split(repeat("y,", 700), ",")) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@row_number_test +POSTHOOK: query: insert into row_number_test select explode(split(repeat("y,", 700), ",")) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@row_number_test +POSTHOOK: Lineage: row_number_test.col SCRIPT [] +col +PREHOOK: query: insert into row_number_test select explode(split(repeat("z,", 600), ",")) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@row_number_test +POSTHOOK: query: insert into row_number_test select explode(split(repeat("z,", 600), ",")) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@row_number_test +POSTHOOK: Lineage: row_number_test.col SCRIPT [] +col +PREHOOK: query: explain select + row_number() over() as r1, + row_number() over(order by col) r2, + row_number() over(partition by col) r3, + row_number() over(partition by col order by col) r4, + row_number() over(partition by 1 order by col) r5, + row_number() over(partition by col order by 2) r6, + row_number() over(partition by 1 order by 2) r7, + col + from row_number_test +PREHOOK: type: QUERY +PREHOOK: Input: default@row_number_test +#### A masked pattern was here #### +POSTHOOK: query: explain select + row_number() over() as r1, + row_number() over(order by col) r2, + row_number() over(partition by col) r3, + row_number() over(partition by col order by col) r4, + row_number() over(partition by 1 order by col) r5, + row_number() over(partition by col order by 2) r6, + row_number() over(partition by 1 order by 2) r7, + col + from row_number_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@row_number_test +#### A masked pattern was here #### +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: row_number_test + Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: col (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 0 ASC NULLS FIRST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_0 (type: int), _col0 (type: string) + outputColumnNames: row_number_window_0, _col0 + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), _col0 (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_0 (type: int) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: row_number_window_1 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_1 (type: int), _col0 (type: int), _col1 (type: string) + outputColumnNames: row_number_window_1, _col0, _col1 + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_1 (type: int), _col0 (type: int) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS FIRST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: row_number_window_2 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: string) + outputColumnNames: row_number_window_2, _col0, _col1, _col2 + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int) + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3 ASC NULLS LAST + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: row_number_window_3 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) + outputColumnNames: row_number_window_3, _col0, _col1, _col2, _col3 + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 1 (type: int), _col3 (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: 1 (type: int) + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int) + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS LAST + partition by: 1 + raw input shape: + window functions: + window function definition + alias: row_number_window_4 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: string) + outputColumnNames: row_number_window_4, _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: string), 2 (type: int) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col4 (type: string) + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int) + Reducer 7 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 2 ASC NULLS LAST + partition by: _col5 + raw input shape: + window functions: + window function definition + alias: row_number_window_5 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: row_number_window_5, _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 1 (type: int), 2 (type: int) + null sort order: az + sort order: ++ + Map-reduce partition columns: 1 (type: int) + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Reducer 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: int, _col6: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 2 ASC NULLS LAST + partition by: 1 + raw input shape: + window functions: + window function definition + alias: row_number_window_6 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: int), _col4 (type: int), _col3 (type: int), _col2 (type: int), _col1 (type: int), _col0 (type: int), row_number_window_6 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: create table row_numbers_vectorized as select +row_number() over() as r1, +row_number() over(order by col) r2, +row_number() over(partition by col) r3, +row_number() over(partition by col order by col) r4, +row_number() over(partition by 1 order by col) r5, +row_number() over(partition by col order by 2) r6, +row_number() over(partition by 1 order by 2) r7, +col +from row_number_test +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@row_number_test +PREHOOK: Output: database:default +PREHOOK: Output: default@row_numbers_vectorized +POSTHOOK: query: create table row_numbers_vectorized as select +row_number() over() as r1, +row_number() over(order by col) r2, +row_number() over(partition by col) r3, +row_number() over(partition by col order by col) r4, +row_number() over(partition by 1 order by col) r5, +row_number() over(partition by col order by 2) r6, +row_number() over(partition by 1 order by 2) r7, +col +from row_number_test +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@row_number_test +POSTHOOK: Output: database:default +POSTHOOK: Output: default@row_numbers_vectorized +POSTHOOK: Lineage: row_numbers_vectorized.col SIMPLE [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r1 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r2 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r3 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r4 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r5 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r6 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_vectorized.r7 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +r1 r2 r3 r4 r5 r6 r7 col +PREHOOK: query: explain select + row_number() over() as r1, + row_number() over(order by col) r2, + row_number() over(partition by col) r3, + row_number() over(partition by col order by col) r4, + row_number() over(partition by 1 order by col) r5, + row_number() over(partition by col order by 2) r6, + row_number() over(partition by 1 order by 2) r7, + col + from row_number_test +PREHOOK: type: QUERY +PREHOOK: Input: default@row_number_test +#### A masked pattern was here #### +POSTHOOK: query: explain select + row_number() over() as r1, + row_number() over(order by col) r2, + row_number() over(partition by col) r3, + row_number() over(partition by col order by col) r4, + row_number() over(partition by 1 order by col) r5, + row_number() over(partition by col order by 2) r6, + row_number() over(partition by 1 order by 2) r7, + col + from row_number_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@row_number_test +#### A masked pattern was here #### +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: row_number_test + Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: col (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 0 ASC NULLS FIRST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_0 (type: int), _col0 (type: string) + outputColumnNames: row_number_window_0, _col0 + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), _col0 (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_0 (type: int) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: row_number_window_1 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_1 (type: int), _col0 (type: int), _col1 (type: string) + outputColumnNames: row_number_window_1, _col0, _col1 + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_1 (type: int), _col0 (type: int) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS FIRST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: row_number_window_2 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: string) + outputColumnNames: row_number_window_2, _col0, _col1, _col2 + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col2 (type: string) + Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3 ASC NULLS LAST + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: row_number_window_3 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) + outputColumnNames: row_number_window_3, _col0, _col1, _col2, _col3 + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 1 (type: int), _col3 (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: 1 (type: int) + Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS LAST + partition by: 1 + raw input shape: + window functions: + window function definition + alias: row_number_window_4 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: string) + outputColumnNames: row_number_window_4, _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: string), 2 (type: int) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col4 (type: string) + Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 2 ASC NULLS LAST + partition by: _col5 + raw input shape: + window functions: + window function definition + alias: row_number_window_5 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: row_number_window_5, _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 1 (type: int), 2 (type: int) + null sort order: az + sort order: ++ + Map-reduce partition columns: 1 (type: int) + Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: int, _col6: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 2 ASC NULLS LAST + partition by: 1 + raw input shape: + window functions: + window function definition + alias: row_number_window_6 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: int), _col4 (type: int), _col3 (type: int), _col2 (type: int), _col1 (type: int), _col0 (type: int), row_number_window_6 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: create table row_numbers_non_vectorized as select +row_number() over() as r1, +row_number() over(order by col) r2, +row_number() over(partition by col) r3, +row_number() over(partition by col order by col) r4, +row_number() over(partition by 1 order by col) r5, +row_number() over(partition by col order by 2) r6, +row_number() over(partition by 1 order by 2) r7, +col +from row_number_test +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@row_number_test +PREHOOK: Output: database:default +PREHOOK: Output: default@row_numbers_non_vectorized +POSTHOOK: query: create table row_numbers_non_vectorized as select +row_number() over() as r1, +row_number() over(order by col) r2, +row_number() over(partition by col) r3, +row_number() over(partition by col order by col) r4, +row_number() over(partition by 1 order by col) r5, +row_number() over(partition by col order by 2) r6, +row_number() over(partition by 1 order by 2) r7, +col +from row_number_test +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@row_number_test +POSTHOOK: Output: database:default +POSTHOOK: Output: default@row_numbers_non_vectorized +POSTHOOK: Lineage: row_numbers_non_vectorized.col SIMPLE [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r1 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r2 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r3 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r4 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r5 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r6 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +POSTHOOK: Lineage: row_numbers_non_vectorized.r7 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] +r1 r2 r3 r4 r5 r6 r7 col +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select exists( +select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_vectorized +minus +select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_non_vectorized +) diff_exists +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Input: default@row_numbers_non_vectorized +PREHOOK: Input: default@row_numbers_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select exists( +select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_vectorized +minus +select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_non_vectorized +) diff_exists +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Input: default@row_numbers_non_vectorized +POSTHOOK: Input: default@row_numbers_vectorized +#### A masked pattern was here #### +diff_exists +false +PREHOOK: query: drop table row_numbers_non_vectorized +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@row_numbers_non_vectorized +PREHOOK: Output: default@row_numbers_non_vectorized +POSTHOOK: query: drop table row_numbers_non_vectorized +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@row_numbers_non_vectorized +POSTHOOK: Output: default@row_numbers_non_vectorized +PREHOOK: query: drop table row_numbers_vectorized +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@row_numbers_vectorized +PREHOOK: Output: default@row_numbers_vectorized +POSTHOOK: query: drop table row_numbers_vectorized +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@row_numbers_vectorized +POSTHOOK: Output: default@row_numbers_vectorized +PREHOOK: query: drop table row_number_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@row_number_test +PREHOOK: Output: default@row_number_test +POSTHOOK: query: drop table row_number_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@row_number_test +POSTHOOK: Output: default@row_number_test diff --git ql/src/test/results/clientpositive/llap/vectorization_multi_value.q.out ql/src/test/results/clientpositive/llap/vectorization_multi_value.q.out new file mode 100644 index 0000000000..8647a791ba --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorization_multi_value.q.out @@ -0,0 +1,677 @@ +PREHOOK: query: DROP TABLE IF EXISTS cond_vector +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS cond_vector +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE cond_vector(a STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cond_vector +POSTHOOK: query: CREATE TABLE cond_vector(a STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cond_vector +PREHOOK: query: INSERT OVERWRITE TABLE cond_vector VALUES("a/b"),("a/b"),("c/d") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cond_vector +POSTHOOK: query: INSERT OVERWRITE TABLE cond_vector VALUES("a/b"),("a/b"),("c/d") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cond_vector +POSTHOOK: Lineage: cond_vector.a SCRIPT [] +PREHOOK: query: SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +{"a":"b"} +{"a":"b"} +{"a":"b"} +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, map('a':'b'), null) (type: map) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:map, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':'b')) -> 3:map) -> 4:map + Statistics: Num rows: 3 Data size: 326 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 326 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, map, map] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +{"a":{"b":"c"}} +{"a":{"b":"c"}} +{"a":{"b":"c"}} +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, map('a':map('b':'c')), null) (type: map>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 4:map>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':map('b':'c')))(children: VectorUDFAdaptor(map('b':'c')) -> 3:map) -> 4:map>) -> 5:map> + Statistics: Num rows: 3 Data size: 241 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 241 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, map, map>, map>] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +{"a":"a/b"} +{"a":"a/b"} +{"a":"c/d"} +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, map('a':a), null) (type: map) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:map, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':a)) -> 3:map) -> 4:map + Statistics: Num rows: 3 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, map, map] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +{"a":{"b":"a/b"}} +{"a":{"b":"a/b"}} +{"a":{"b":"c/d"}} +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, map('a':map('b':a)), null) (type: map>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 4:map>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':map('b':a)))(children: VectorUDFAdaptor(map('b':a)) -> 3:map) -> 4:map>) -> 5:map> + Statistics: Num rows: 3 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, map, map>, map>] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +["a","b"] +["a","b"] +["a","b"] +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, array('a','b'), null) (type: array) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:array, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array('a','b')) -> 3:array) -> 4:array + Statistics: Num rows: 3 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, array, array] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +[["a","b"],["c","d"]] +[["a","b"],["c","d"]] +[["a","b"],["c","d"]] +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, array(array('a','b'),array('c','d')), null) (type: array>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 5:array>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array(array('a','b'),array('c','d')))(children: VectorUDFAdaptor(array('a','b')) -> 3:array, VectorUDFAdaptor(array('c','d')) -> 4:array) -> 5:array>) -> 6:array> + Statistics: Num rows: 3 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, array, array, array>, array>] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +["a","a/b"] +["a","a/b"] +["a","c/d"] +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, array('a',a), null) (type: array) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:array, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array('a',a)) -> 3:array) -> 4:array + Statistics: Num rows: 3 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, array, array] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +[["a","a/b"],["b","c"]] +[["a","a/b"],["b","c"]] +[["a","c/d"],["b","c"]] +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector +PREHOOK: type: QUERY +PREHOOK: Input: default@cond_vector +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cond_vector +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cond_vector + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] + Select Operator + expressions: if(true, array(array('a',a),array('b','c')), null) (type: array>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [6] + selectExpressions: IfExprCondExprNull(col 2:boolean, col 5:array>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array(array('a',a),array('b','c')))(children: VectorUDFAdaptor(array('a',a)) -> 3:array, VectorUDFAdaptor(array('b','c')) -> 4:array) -> 5:array>) -> 6:array> + Statistics: Num rows: 3 Data size: 19280 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 19280 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: true + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: a:string + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, array, array, array>, array>] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out new file mode 100644 index 0000000000..e741bbdd2c --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorization_numeric_overflows.q.out @@ -0,0 +1,1366 @@ +PREHOOK: query: CREATE TABLE test_overflow ( + ctinyint1 TINYINT, + ctinyint2 TINYINT, + csmallint1 SMALLINT, + csmallint2 SMALLINT, + cint1 INT, + cint2 INT, + cbigint1 BIGINT, + cbigint2 BIGINT, + cfloat1 FLOAT, + cfloat2 FLOAT, + cdouble1 DOUBLE, + cdouble2 DOUBLE) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_overflow +POSTHOOK: query: CREATE TABLE test_overflow ( + ctinyint1 TINYINT, + ctinyint2 TINYINT, + csmallint1 SMALLINT, + csmallint2 SMALLINT, + cint1 INT, + cint2 INT, + cbigint1 BIGINT, + cbigint2 BIGINT, + cfloat1 FLOAT, + cfloat2 FLOAT, + cdouble1 DOUBLE, + cdouble2 DOUBLE) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_overflow +PREHOOK: query: insert into test_overflow values (-128, 127, -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, 1.401298464324817E-45, 3.4028234663852886E38, 4.9E-324, 1.7976931348623157E308) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_overflow +POSTHOOK: query: insert into test_overflow values (-128, 127, -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, 1.401298464324817E-45, 3.4028234663852886E38, 4.9E-324, 1.7976931348623157E308) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_overflow +POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] +PREHOOK: query: insert into test_overflow values (127, -128, 32767, -32768, 2147483647, -2147483648, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_overflow +POSTHOOK: query: insert into test_overflow values (127, -128, 32767, -32768, 2147483647, -2147483648, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_overflow +POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] +PREHOOK: query: insert into test_overflow values (64, 65, 32767, -32768, 1073741824, 1073741825, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_overflow +POSTHOOK: query: insert into test_overflow values (64, 65, 32767, -32768, 1073741824, 1073741825, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_overflow +POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] +POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] +PREHOOK: query: select * from test_overflow order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select * from test_overflow order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 +64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +PREHOOK: query: explain vectorization expression +select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((cint1 - 2) > 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColGreaterLongScalar(col 13:int, val 0)(children: LongColSubtractLongScalarChecked(col 4:int, val 2) -> 13:int) + predicate: ((cint1 - 2) > 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint1 (type: int), (cint1 - 2) (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [4, 14] + selectExpressions: LongColSubtractLongScalarChecked(col 4:int, val 2) -> 14:int + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-2147483648 2147483646 +1073741824 1073741822 +2147483647 2147483645 +PREHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-2147483648 2147483646 +1073741824 1073741822 +2147483647 2147483645 +PREHOOK: query: explain vectorization expression +select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((cint2 + 2) < 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColLessLongScalar(col 13:int, val 0)(children: LongColAddLongScalarChecked(col 5:int, val 2) -> 13:int) + predicate: ((cint2 + 2) < 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint2 (type: int), (cint2 + 2) (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5, 14] + selectExpressions: LongColAddLongScalarChecked(col 5:int, val 2) -> 14:int + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-2147483648 -2147483646 +2147483647 -2147483647 +PREHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-2147483648 -2147483646 +2147483647 -2147483647 +PREHOOK: query: explain vectorization expression +select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((cint2 * 2) < 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColLessLongScalar(col 13:int, val 0)(children: LongColMultiplyLongScalarChecked(col 5:int, val 2) -> 13:int) + predicate: ((cint2 * 2) < 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint2 (type: int), (cint2 * 2) (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [5, 14] + selectExpressions: LongColMultiplyLongScalarChecked(col 5:int, val 2) -> 14:int + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +1073741825 -2147483646 +2147483647 -2 +PREHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +1073741825 -2147483646 +2147483647 -2 +PREHOOK: query: explain vectorization expression +select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((ctinyint1 - 2Y) > 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColGreaterLongScalar(col 13:tinyint, val 0)(children: LongColSubtractLongScalarChecked(col 0:tinyint, val 2) -> 13:tinyint) + predicate: ((ctinyint1 - 2Y) > 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint1 (type: tinyint), (ctinyint1 - 2Y) (type: tinyint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 14] + selectExpressions: LongColSubtractLongScalarChecked(col 0:tinyint, val 2) -> 14:tinyint + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: tinyint) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: tinyint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 126 +64 62 +127 125 +PREHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 126 +64 62 +127 125 +PREHOOK: query: explain vectorization expression +select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((ctinyint2 + 2Y) < 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColLessLongScalar(col 13:tinyint, val 0)(children: LongColAddLongScalarChecked(col 1:tinyint, val 2) -> 13:tinyint) + predicate: ((ctinyint2 + 2Y) < 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint2 (type: tinyint), (ctinyint2 + 2) (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 14] + selectExpressions: LongColAddLongScalarChecked(col 1:tinyint, val 2) -> 14:int + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 -126 +127 129 +PREHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 -126 +127 129 +PREHOOK: query: explain vectorization expression +select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((csmallint2 * 2S) < 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColLessLongScalar(col 13:smallint, val 0)(children: LongColMultiplyLongScalarChecked(col 3:smallint, val 2) -> 13:smallint) + predicate: ((csmallint2 * 2S) < 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: csmallint2 (type: smallint), (csmallint2 * 2) (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [3, 14] + selectExpressions: LongColMultiplyLongScalarChecked(col 3:smallint, val 2) -> 14:int + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: smallint) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +32767 65534 +PREHOOK: query: explain vectorization expression +select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((csmallint2 * 2S) < 0) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((csmallint2 * 2S) < 0) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: csmallint2 (type: smallint), (csmallint2 * 2) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: smallint) + null sort order: z + sort order: + + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: all inputs (cache only) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +32767 65534 +PREHOOK: query: create table parquettable (t1 tinyint, t2 tinyint, i1 int, i2 int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquettable +POSTHOOK: query: create table parquettable (t1 tinyint, t2 tinyint, i1 int, i2 int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquettable +PREHOOK: query: insert into parquettable values (-104, 25,2147483647, 10), (-112, 24, -2147483648, 10), (54, 9, 2147483647, -50) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@parquettable +POSTHOOK: query: insert into parquettable values (-104, 25,2147483647, 10), (-112, 24, -2147483648, 10), (54, 9, 2147483647, -50) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@parquettable +POSTHOOK: Lineage: parquettable.i1 SCRIPT [] +POSTHOOK: Lineage: parquettable.i2 SCRIPT [] +POSTHOOK: Lineage: parquettable.t1 SCRIPT [] +POSTHOOK: Lineage: parquettable.t2 SCRIPT [] +PREHOOK: query: explain vectorization expression select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquettable + filterExpr: ((t1 - t2) < 50) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t1 - t2) < 50) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 - t2) (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: tinyint) + null sort order: z + sort order: - + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: tinyint) + Execution mode: llap + LLAP IO: all inputs (cache only) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), KEY.reducesinkkey0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +54 9 45 +PREHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +54 9 45 +PREHOOK: query: explain vectorization expression select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquettable + filterExpr: ((i1 - i2) < 50) (type: boolean) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColLessLongScalar(col 5:int, val 50)(children: LongColSubtractLongColumnChecked(col 2:int, col 3:int) -> 5:int) + predicate: ((i1 - i2) < 50) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: i1 (type: int), i2 (type: int), (i1 - i2) (type: int) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 6] + selectExpressions: LongColSubtractLongColumnChecked(col 2:int, col 3:int) -> 6:int + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + null sort order: z + sort order: - + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 2, 0] + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +2147483647 -50 -2147483599 +PREHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettable +#### A masked pattern was here #### +POSTHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettable +#### A masked pattern was here #### +2147483647 -50 -2147483599 +PREHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PREHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PREHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PREHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PREHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +2147483647 0 +PREHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +2147483647 0 +PREHOOK: query: select 2147483648 from test_overflow +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select 2147483648 from test_overflow +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +2147483648 +2147483648 +2147483648 +PREHOOK: query: select 2147483648 from test_overflow +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select 2147483648 from test_overflow +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +2147483648 +2147483648 +2147483648 +PREHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 +127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +PREHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 +127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +PREHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 +64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +PREHOOK: query: explain vectorization expression select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_overflow + filterExpr: ((csmallint1 pmod 16385S) > 0) (type: boolean) + Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColGreaterLongScalar(col 13:smallint, val 0)(children: PosModLongToLong(col 2, divisor 16385) -> 13:smallint) + predicate: ((csmallint1 pmod 16385S) > 0) (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint1 (type: tinyint), ctinyint2 (type: tinyint), csmallint1 (type: smallint), csmallint2 (type: smallint), cint1 (type: int), cint2 (type: int), cbigint1 (type: bigint), cbigint2 (type: bigint), cfloat1 (type: float), cfloat2 (type: float), cdouble1 (type: double), cdouble2 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: tinyint), _col2 (type: smallint), _col3 (type: smallint), _col4 (type: int), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: float), _col9 (type: float), _col10 (type: double), _col11 (type: double) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: smallint), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: bigint), VALUE._col7 (type: float), VALUE._col8 (type: float), VALUE._col9 (type: double), VALUE._col10 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_overflow +#### A masked pattern was here #### +POSTHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_overflow +#### A masked pattern was here #### +-128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 +64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 +127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 diff --git ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out new file mode 100644 index 0000000000..ab732bd9ba --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out @@ -0,0 +1,208 @@ +WARNING: Comparing a bigint and a double may result in a loss of precision. +PREHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + filterExpr: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) + Statistics: Num rows: 12288 Data size: 183488 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) + Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Offset of rows: 3 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: 2 + Processor Tree: + ListSink + +WARNING: Comparing a bigint and a double may result in a loss of precision. +PREHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +-1887561756 10361.0 +-1887561756 -8881.0 +PREHOOK: query: explain vectorization expression +select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + filterExpr: ctinyint is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 146796 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:tinyint) + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: ++ + keys: ctinyint (type: tinyint), cdouble (type: double) + null sort order: zz + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + top n: 13 + Top N Key Vectorization: + className: VectorTopNKeyOperator + keyExpressions: col 0:tinyint, col 5:double + native: true + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 5, 1] + Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: smallint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), VALUE._col0 (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1, 2] + Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Limit Vectorization: + className: VectorLimitOperator + native: true + Offset of rows: 10 + Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 3 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +-64 -7196.0 -7196 +-64 -6907.0 -6907 +-64 -4803.0 -4803 diff --git ql/src/test/results/clientpositive/vectorization_parquet_ppd_decimal.q.out ql/src/test/results/clientpositive/llap/vectorization_parquet_ppd_decimal.q.out similarity index 94% rename from ql/src/test/results/clientpositive/vectorization_parquet_ppd_decimal.q.out rename to ql/src/test/results/clientpositive/llap/vectorization_parquet_ppd_decimal.q.out index 9cbed4ff4e..a86113b909 100644 --- ql/src/test/results/clientpositive/vectorization_parquet_ppd_decimal.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_parquet_ppd_decimal.q.out @@ -6,11 +6,21 @@ POSTHOOK: query: create table newtypestbl_n1(c char(10), v varchar(10), d decima POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@newtypestbl_n1 -PREHOOK: query: insert overwrite table newtypestbl_n1 select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 10) uniontbl +PREHOOK: query: insert overwrite table newtypestbl_n1 +select * + from (select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 limit 5) u1 + union all + select * from (select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 5) u2 + ) uniontbl PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@newtypestbl_n1 -POSTHOOK: query: insert overwrite table newtypestbl_n1 select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 10) uniontbl +POSTHOOK: query: insert overwrite table newtypestbl_n1 +select * + from (select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("1970-02-20" as date) from src src1 limit 5) u1 + union all + select * from (select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1970-02-27" as date) from src src2 limit 5) u2 + ) uniontbl POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@newtypestbl_n1 @@ -26,11 +36,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d=0.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 @@ -39,11 +49,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 @@ -52,11 +62,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 @@ -65,11 +75,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 @@ -78,11 +88,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d=cast('0.22' as float) POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 @@ -91,11 +101,11 @@ POSTHOOK: query: select * from newtypestbl_n1 where d=cast('0.22' as float) POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypestbl_n1 #### A masked pattern was here #### -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 -apple bee 0.220 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 +apple bee 0.22 1970-02-20 PREHOOK: query: select * from newtypestbl_n1 where d!=0.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypestbl_n1 diff --git ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out new file mode 100644 index 0000000000..064bce7b6b --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorization_parquet_projection.q.out @@ -0,0 +1,722 @@ +PREHOOK: query: DROP TABLE IF EXISTS parquet_types_staging_n0 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS parquet_types_staging_n0 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_types_staging_n0 ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string, + t timestamp, + cchar char(5), + cvarchar varchar(10), + cbinary string, + m1 map, + l1 array, + st1 struct, + d date +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_types_staging_n0 +POSTHOOK: query: CREATE TABLE parquet_types_staging_n0 ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string, + t timestamp, + cchar char(5), + cvarchar varchar(10), + cbinary string, + m1 map, + l1 array, + st1 struct, + d date +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_types_staging_n0 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging_n0 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@parquet_types_staging_n0 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging_n0 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@parquet_types_staging_n0 +PREHOOK: query: DROP TABLE IF EXISTS parquet_project_test +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS parquet_project_test +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_project_test( +cint int, +m1 map +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_project_test +POSTHOOK: query: CREATE TABLE parquet_project_test( +cint int, +m1 map +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_project_test +PREHOOK: query: insert into parquet_project_test +select ctinyint, map("color","red") from parquet_types_staging_n0 +where ctinyint = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_types_staging_n0 +PREHOOK: Output: default@parquet_project_test +POSTHOOK: query: insert into parquet_project_test +select ctinyint, map("color","red") from parquet_types_staging_n0 +where ctinyint = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_types_staging_n0 +POSTHOOK: Output: default@parquet_project_test +POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] +POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] +PREHOOK: query: insert into parquet_project_test +select ctinyint, map("color","green") from parquet_types_staging_n0 +where ctinyint = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_types_staging_n0 +PREHOOK: Output: default@parquet_project_test +POSTHOOK: query: insert into parquet_project_test +select ctinyint, map("color","green") from parquet_types_staging_n0 +where ctinyint = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_types_staging_n0 +POSTHOOK: Output: default@parquet_project_test +POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] +POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] +PREHOOK: query: insert into parquet_project_test +select ctinyint, map("color","blue") from parquet_types_staging_n0 +where ctinyint = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_types_staging_n0 +PREHOOK: Output: default@parquet_project_test +POSTHOOK: query: insert into parquet_project_test +select ctinyint, map("color","blue") from parquet_types_staging_n0 +where ctinyint = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_types_staging_n0 +POSTHOOK: Output: default@parquet_project_test +POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] +POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] +PREHOOK: query: explain vectorization select * from parquet_project_test +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select * from parquet_project_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_project_test + Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), m1 (type: map) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type MAP not enabled (map) since hive.vectorized.complex.types.enabled IS false + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from parquet_project_test +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: select * from parquet_project_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +1 {"color":"red"} +2 {"color":"green"} +2 {"color":"green"} +2 {"color":"green"} +2 {"color":"green"} +2 {"color":"green"} +2 {"color":"green"} +2 {"color":"green"} +3 {"color":"blue"} +3 {"color":"blue"} +3 {"color":"blue"} +3 {"color":"blue"} +3 {"color":"blue"} +3 {"color":"blue"} +3 {"color":"blue"} +PREHOOK: query: explain vectorization select count(*) from parquet_project_test +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select count(*) from parquet_project_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_project_test + Statistics: Num rows: 22 Data size: 553 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 22 Data size: 553 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.95454544 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from parquet_project_test +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from parquet_project_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +22 +PREHOOK: query: explain vectorization select cint, count(*) from parquet_project_test +group by cint +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select cint, count(*) from parquet_project_test +group by cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_project_test + Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: cint (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, count(*) from parquet_project_test +group by cint +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: select cint, count(*) from parquet_project_test +group by cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +1 8 +2 7 +3 7 +PREHOOK: query: explain vectorization select m1["color"], count(*) from parquet_project_test +group by m1["color"] +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select m1["color"], count(*) from parquet_project_test +group by m1["color"] +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_project_test + Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: m1['color'] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type MAP not enabled (map) since hive.vectorized.complex.types.enabled IS false + vectorized: false + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 10120 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 10120 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select m1["color"], count(*) from parquet_project_test +group by m1["color"] +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +POSTHOOK: query: select m1["color"], count(*) from parquet_project_test +group by m1["color"] +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_project_test +#### A masked pattern was here #### +blue 7 +green 7 +red 8 +PREHOOK: query: create table if not exists parquet_nullsplit(key string, val string) partitioned by (len string) +stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_nullsplit +POSTHOOK: query: create table if not exists parquet_nullsplit(key string, val string) partitioned by (len string) +stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_nullsplit +PREHOOK: query: insert into table parquet_nullsplit partition(len='1') +values ('one', 'red') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@parquet_nullsplit@len=1 +POSTHOOK: query: insert into table parquet_nullsplit partition(len='1') +values ('one', 'red') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@parquet_nullsplit@len=1 +POSTHOOK: Lineage: parquet_nullsplit PARTITION(len=1).key SCRIPT [] +POSTHOOK: Lineage: parquet_nullsplit PARTITION(len=1).val SCRIPT [] +PREHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_nullsplit +PREHOOK: Input: default@parquet_nullsplit@len=1 +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_nullsplit +POSTHOOK: Input: default@parquet_nullsplit@len=1 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_nullsplit + filterExpr: (len = '1') (type: boolean) + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from parquet_nullsplit where len = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_nullsplit +PREHOOK: Input: default@parquet_nullsplit@len=1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from parquet_nullsplit where len = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_nullsplit +POSTHOOK: Input: default@parquet_nullsplit@len=1 +#### A masked pattern was here #### +1 +PREHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '99' +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_nullsplit +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '99' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_nullsplit +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquet_nullsplit + filterExpr: (len = '99') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (len = '99') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: unknown + Map Vectorization: + enabled: true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from parquet_nullsplit where len = '99' +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_nullsplit +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from parquet_nullsplit where len = '99' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_nullsplit +#### A masked pattern was here #### +0 +PREHOOK: query: drop table parquet_nullsplit +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parquet_nullsplit +PREHOOK: Output: default@parquet_nullsplit +POSTHOOK: query: drop table parquet_nullsplit +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parquet_nullsplit +POSTHOOK: Output: default@parquet_nullsplit +PREHOOK: query: drop table parquet_project_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parquet_project_test +PREHOOK: Output: default@parquet_project_test +POSTHOOK: query: drop table parquet_project_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parquet_project_test +POSTHOOK: Output: default@parquet_project_test +PREHOOK: query: drop table parquet_types_staging_n0 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parquet_types_staging_n0 +PREHOOK: Output: default@parquet_types_staging_n0 +POSTHOOK: query: drop table parquet_types_staging_n0 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parquet_types_staging_n0 +POSTHOOK: Output: default@parquet_types_staging_n0 diff --git ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out ql/src/test/results/clientpositive/llap/vectorization_sum_if_when.q.out similarity index 95% rename from ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out rename to ql/src/test/results/clientpositive/llap/vectorization_sum_if_when.q.out index 84dccd6bb2..4d6e619a37 100644 --- ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out +++ ql/src/test/results/clientpositive/llap/vectorization_sum_if_when.q.out @@ -31,9 +31,9 @@ POSTHOOK: query: create table vectorization_sum_if_when_b (x int) stored as orc POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@vectorization_sum_if_when_b -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[27][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: insert into table vectorization_sum_if_when_b select least(t1.x + t2.x + t3.x + t4.x, 1) from vectorization_sum_if_when_a t1, vectorization_sum_if_when_a t2, vectorization_sum_if_when_a t3, vectorization_sum_if_when_a t4 PREHOOK: type: QUERY PREHOOK: Input: default@vectorization_sum_if_when_a diff --git ql/src/test/results/clientpositive/llap/vectorized_join46_mr.q.out ql/src/test/results/clientpositive/llap/vectorized_join46_mr.q.out new file mode 100644 index 0000000000..ca65653467 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorized_join46_mr.q.out @@ -0,0 +1,2689 @@ +PREHOOK: query: CREATE TABLE test1 (key INT, value INT, col_1 STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test1 +POSTHOOK: query: CREATE TABLE test1 (key INT, value INT, col_1 STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test1 +PREHOOK: query: INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'), + (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test1 +POSTHOOK: query: INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'), + (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test1 +POSTHOOK: Lineage: test1.col_1 SCRIPT [] +POSTHOOK: Lineage: test1.key SCRIPT [] +POSTHOOK: Lineage: test1.value SCRIPT [] +col1 col2 col3 +PREHOOK: query: CREATE TABLE test2 (key INT, value INT, col_2 STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test2 +POSTHOOK: query: CREATE TABLE test2 (key INT, value INT, col_2 STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test2 +PREHOOK: query: INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'), + (104, 3, 'Fli'), (105, NULL, 'None') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test2 +POSTHOOK: query: INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'), + (104, 3, 'Fli'), (105, NULL, 'None') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test2 +POSTHOOK: Lineage: test2.col_2 SCRIPT [] +POSTHOOK: Lineage: test2.key SCRIPT [] +POSTHOOK: Lineage: test2.value SCRIPT [] +col1 col2 col3 +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + Map Join Vectorization: + className: VectorMapJoinOuterLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + input vertices: + 1 Map 2 + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob NULL NULL NULL +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None NULL NULL NULL +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND test1.key between 100 and 102 + AND test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND test1.key between 100 and 102 + AND test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Vectorization: + className: VectorSelectOperator + native: true + selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + Map Join Vectorization: + className: VectorMapJoinOuterLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 + input vertices: + 1 Map 2 + Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: (value is not null and key BETWEEN 100 AND 102) (type: boolean) + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: (value is not null and key BETWEEN 100 AND 102) (type: boolean) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND test1.key between 100 and 102 + AND test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND test1.key between 100 and 102 + AND test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob NULL NULL NULL +101 2 Car 102 2 Del +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat NULL NULL NULL +NULL NULL None NULL NULL NULL +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102 + AND test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102 + AND test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Vectorization: + className: VectorSelectOperator + native: true + selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + filter predicates: + 0 {_col3} + 1 + keys: + 0 + 1 + Map Join Vectorization: + className: VectorMapJoinOuterFilteredOperator + native: false + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nativeConditionsNotMet: Outer Join has keys IS false + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 + input vertices: + 1 Map 2 + Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102 + AND test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102 + AND test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +101 2 Car 102 2 Del +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat NULL NULL NULL +NULL NULL None NULL NULL NULL +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value AND true) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value AND true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + Map Join Vectorization: + className: VectorMapJoinOuterLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + input vertices: + 0 Map 1 + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value AND true) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value AND true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL NULL 104 3 Fli +NULL NULL NULL 105 NULL None +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Vectorization: + className: VectorSelectOperator + native: true + selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + filter predicates: + 0 {_col3} + 1 + keys: + 0 + 1 + Map Join Vectorization: + className: VectorMapJoinOuterFilteredOperator + native: false + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nativeConditionsNotMet: Outer Join has keys IS false + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 + input vertices: + 1 Map 2 + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat NULL NULL NULL +NULL NULL None NULL NULL NULL +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 1 Map 2 + residual filter predicates: {((_col1 = _col5) or _col3 or _col4 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None 102 2 Del +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 1 Map 2 + residual filter predicates: {((_col1 = _col5) or _col3)} + Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None NULL NULL NULL +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + input vertices: + 1 Map 2 + residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None 102 2 Del +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 1 Map 2 + residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + Map 2 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 LEFT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob NULL NULL NULL +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat 102 2 Del +NULL NULL None NULL NULL NULL +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 0 Map 1 + residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col6)} + Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None 102 2 Del +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + input vertices: + 0 Map 1 + residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 0 Map 1 + residual filter predicates: {((_col1 = _col4) or _col6)} + Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL NULL 104 3 Fli +NULL NULL NULL 105 NULL None +NULL NULL None 102 2 Del +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 2 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + input vertices: + 0 Map 1 + residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} + Statistics: Num rows: 7 Data size: 1265 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: MAPJOIN operator: Non-equi joins not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 RIGHT OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +99 2 Mat 102 2 Del +NULL NULL NULL 104 3 Fli +NULL NULL NULL 105 NULL None +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102 + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None 102 2 Del +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test1.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +100 1 Bob 103 2 Ema +100 1 Bob 104 3 Fli +100 1 Bob 105 NULL None +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +101 2 Car 104 3 Fli +101 2 Car 105 NULL None +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL None NULL NULL NULL +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + OR test2.key between 100 and 102) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob 102 2 Del +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None 102 2 Del +99 0 Alice 102 2 Del +99 2 Mat 102 2 Del +99 2 Mat 103 2 Ema +NULL NULL NULL 104 3 Fli +NULL NULL NULL 105 NULL None +NULL NULL None 102 2 Del +PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR +SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Full Outer Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} + Statistics: Num rows: 13 Data size: 1711 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 1711 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM test1 FULL OUTER JOIN test2 +ON (test1.value=test2.value + AND (test1.key between 100 and 102 + OR test2.key between 100 and 102)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 +100 1 Bob NULL NULL NULL +101 2 Car 102 2 Del +101 2 Car 103 2 Ema +98 NULL None NULL NULL NULL +99 0 Alice NULL NULL NULL +99 2 Mat 102 2 Del +NULL NULL NULL 104 3 Fli +NULL NULL NULL 105 NULL None +NULL NULL None NULL NULL NULL diff --git ql/src/test/results/clientpositive/llap/vectorized_mapjoin2.q.out ql/src/test/results/clientpositive/llap/vectorized_mapjoin2.q.out new file mode 100644 index 0000000000..c33009283e --- /dev/null +++ ql/src/test/results/clientpositive/llap/vectorized_mapjoin2.q.out @@ -0,0 +1,226 @@ +PREHOOK: query: create temporary table x (a int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@x +POSTHOOK: query: create temporary table x (a int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@x +PREHOOK: query: create temporary table y (b int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@y +POSTHOOK: query: create temporary table y (b int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@y +PREHOOK: query: insert into x values(1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values(1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +POSTHOOK: Lineage: x.a SCRIPT [] +PREHOOK: query: insert into y values(1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@y +POSTHOOK: query: insert into y values(1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@y +POSTHOOK: Lineage: y.b SCRIPT [] +PREHOOK: query: explain vectorization expression +select count(1) from x, y where a = b +PREHOOK: type: QUERY +PREHOOK: Input: default@x +PREHOOK: Input: default@y +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization expression +select count(1) from x, y where a = b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@x +POSTHOOK: Input: default@y +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + filterExpr: a is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: a is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Map Join Vectorization: + className: VectorMapJoinInnerBigOnlyLongOperator + native: true + nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Fast Hash Table and No Hybrid Hash Join IS true + hashTableImplementationType: FAST + input vertices: + 1 Map 3 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + Group By Vectorization: + aggregators: VectorUDAFCountStar(*) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: y + filterExpr: b is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: b is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: b (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) from x, y where a = b +PREHOOK: type: QUERY +PREHOOK: Input: default@x +PREHOOK: Input: default@y +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from x, y where a = b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@x +POSTHOOK: Input: default@y +#### A masked pattern was here #### +1 diff --git ql/src/test/results/clientpositive/view_alias.q.out ql/src/test/results/clientpositive/llap/view_alias.q.out similarity index 89% rename from ql/src/test/results/clientpositive/view_alias.q.out rename to ql/src/test/results/clientpositive/llap/view_alias.q.out index 44de7b3bd2..d8bac2450a 100644 --- ql/src/test/results/clientpositive/view_alias.q.out +++ ql/src/test/results/clientpositive/llap/view_alias.q.out @@ -2,18 +2,18 @@ PREHOOK: query: drop view v_n6 PREHOOK: type: DROPVIEW POSTHOOK: query: drop view v_n6 POSTHOOK: type: DROPVIEW -PREHOOK: query: create view v_n6 as select key, '12' from src +PREHOOK: query: create view v_n6 as select 10 - key, '12' from src PREHOOK: type: CREATEVIEW PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@v_n6 -POSTHOOK: query: create view v_n6 as select key, '12' from src +POSTHOOK: query: create view v_n6 as select 10 - key, '12' from src POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v_n6 +POSTHOOK: Lineage: v_n6._c0 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: v_n6._c1 SIMPLE [] -POSTHOOK: Lineage: v_n6.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] PREHOOK: query: desc formatted v_n6 PREHOOK: type: DESCTABLE PREHOOK: Input: default@v_n6 @@ -21,7 +21,7 @@ POSTHOOK: query: desc formatted v_n6 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@v_n6 # col_name data_type comment -key string +_c0 double _c1 string # Detailed Table Information @@ -43,23 +43,23 @@ Bucket Columns: [] Sort Columns: [] # View Information -Original Query: select key, '12' from src -Expanded Query: select `src`.`key`, '12' from `default`.`src` -PREHOOK: query: select * from v_n6 order by `_c1` limit 5 +Original Query: select 10 - key, '12' from src +Expanded Query: select 10 - `src`.`key`, '12' from `default`.`src` +PREHOOK: query: select * from v_n6 order by `_c0`, `_c1` limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@v_n6 #### A masked pattern was here #### -POSTHOOK: query: select * from v_n6 order by `_c1` limit 5 +POSTHOOK: query: select * from v_n6 order by `_c0`, `_c1` limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@v_n6 #### A masked pattern was here #### -238 12 -86 12 -311 12 -27 12 -165 12 +-488.0 12 +-488.0 12 +-488.0 12 +-487.0 12 +-486.0 12 PREHOOK: query: drop view v_n6 PREHOOK: type: DROPVIEW PREHOOK: Input: default@v_n6 @@ -179,21 +179,21 @@ Sort Columns: [] # View Information Original Query: select *, '12' from src Expanded Query: select `src`.`key`, `src`.`value`, '12' from `default`.`src` -PREHOOK: query: select * from v_n6 order by `_c2` limit 5 +PREHOOK: query: select * from v_n6 order by key, value, `_c2` limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@v_n6 #### A masked pattern was here #### -POSTHOOK: query: select * from v_n6 order by `_c2` limit 5 +POSTHOOK: query: select * from v_n6 order by key, value, `_c2` limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@v_n6 #### A masked pattern was here #### -238 val_238 12 -86 val_86 12 -311 val_311 12 -27 val_27 12 -165 val_165 12 +0 val_0 12 +0 val_0 12 +0 val_0 12 +10 val_10 12 +100 val_100 12 PREHOOK: query: drop view v_n6 PREHOOK: type: DROPVIEW PREHOOK: Input: default@v_n6 @@ -247,21 +247,21 @@ Sort Columns: [] # View Information Original Query: select *, '12' as _c121 from src Expanded Query: select `src`.`key`, `src`.`value`, '12' as `_c121` from `default`.`src` -PREHOOK: query: select * from v_n6 order by `_c121` limit 5 +PREHOOK: query: select * from v_n6 order by key, value, `_c121` limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@v_n6 #### A masked pattern was here #### -POSTHOOK: query: select * from v_n6 order by `_c121` limit 5 +POSTHOOK: query: select * from v_n6 order by key, value, `_c121` limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@v_n6 #### A masked pattern was here #### -238 val_238 12 -86 val_86 12 -311 val_311 12 -27 val_27 12 -165 val_165 12 +0 val_0 12 +0 val_0 12 +0 val_0 12 +10 val_10 12 +100 val_100 12 PREHOOK: query: drop view v_n6 PREHOOK: type: DROPVIEW PREHOOK: Input: default@v_n6 @@ -313,21 +313,21 @@ Sort Columns: [] # View Information Original Query: select key, count(*) from src group by key Expanded Query: select `src`.`key`, count(*) from `default`.`src` group by `src`.`key` -PREHOOK: query: select * from v_n6 order by `_c1` limit 5 +PREHOOK: query: select * from v_n6 order by key, `_c1` limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@v_n6 #### A masked pattern was here #### -POSTHOOK: query: select * from v_n6 order by `_c1` limit 5 +POSTHOOK: query: select * from v_n6 order by key, `_c1` limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@v_n6 #### A masked pattern was here #### -11 1 -105 1 -114 1 -96 1 +0 3 10 1 +100 2 +103 2 +104 2 PREHOOK: query: drop view v_n6 PREHOOK: type: DROPVIEW PREHOOK: Input: default@v_n6 @@ -352,21 +352,21 @@ POSTHOOK: query: create table b_n7 (cb_n7 string, cbb_n7 string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@b_n7 -PREHOOK: query: insert into a_n9 select * from src limit 5 +PREHOOK: query: insert into a_n9 select * from src order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@a_n9 -POSTHOOK: query: insert into a_n9 select * from src limit 5 +POSTHOOK: query: insert into a_n9 select * from src order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@a_n9 POSTHOOK: Lineage: a_n9.ca_n9 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: a_n9.caa_n9 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert into b_n7 select * from src limit 5 +PREHOOK: query: insert into b_n7 select * from src order by key, value limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@b_n7 -POSTHOOK: query: insert into b_n7 select * from src limit 5 +POSTHOOK: query: insert into b_n7 select * from src order by key, value limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@b_n7 @@ -427,20 +427,20 @@ Sort Columns: [] # View Information Original Query: select '010', a_n9.*, 121, b_n7.*, 234 from a_n9 join b_n7 on a_n9.ca_n9 = b_n7.cb_n7 Expanded Query: select '010', `a_n9`.`ca_n9`, `a_n9`.`caa_n9`, 121, `b_n7`.`cb_n7`, `b_n7`.`cbb_n7`, 234 from `default`.`a_n9` join `default`.`b_n7` on `a_n9`.`ca_n9` = `b_n7`.`cb_n7` -PREHOOK: query: select * from v_n6 order by `_c3` limit 5 +PREHOOK: query: select * from v_n6 order by `_c3`, `_c0`, ca_n9, caa_n9, cb_n7, cbb_n7 limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@a_n9 PREHOOK: Input: default@b_n7 PREHOOK: Input: default@v_n6 #### A masked pattern was here #### -POSTHOOK: query: select * from v_n6 order by `_c3` limit 5 +POSTHOOK: query: select * from v_n6 order by `_c3`, `_c0`, ca_n9, caa_n9, cb_n7, cbb_n7 limit 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@a_n9 POSTHOOK: Input: default@b_n7 POSTHOOK: Input: default@v_n6 #### A masked pattern was here #### -010 165 val_165 121 165 val_165 234 -010 238 val_238 121 238 val_238 234 -010 27 val_27 121 27 val_27 234 -010 311 val_311 121 311 val_311 234 -010 86 val_86 121 86 val_86 234 +010 0 val_0 121 0 val_0 234 +010 0 val_0 121 0 val_0 234 +010 0 val_0 121 0 val_0 234 +010 0 val_0 121 0 val_0 234 +010 0 val_0 121 0 val_0 234 diff --git ql/src/test/results/clientpositive/view_cbo.q.out ql/src/test/results/clientpositive/llap/view_cbo.q.out similarity index 66% rename from ql/src/test/results/clientpositive/view_cbo.q.out rename to ql/src/test/results/clientpositive/llap/view_cbo.q.out index 2c07c723bd..36878428e5 100644 --- ql/src/test/results/clientpositive/view_cbo.q.out +++ ql/src/test/results/clientpositive/llap/view_cbo.q.out @@ -14,81 +14,86 @@ POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string), key (type: string), (UDFToDouble(key) + 1.0D) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 1.0D) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col2), count(_col2) + keys: _col0 (type: string), _col1 (type: string), 0L (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 750 Data size: 151500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + null sort order: zzz + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 750 Data size: 151500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: double), _col4 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: sum(_col2), count(_col2) - keys: _col0 (type: string), _col1 (type: string), 0L (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + aggregations: sum(VALUE._col0), count(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 750 Data size: 151500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) - null sort order: zzz - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + pruneGroupingSetId: true + Top N Key Operator + sort order: ++ + keys: _col1 (type: string), _col0 (type: string) + null sort order: zz Statistics: Num rows: 750 Data size: 151500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: double), _col4 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 750 Data size: 151500 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true - Select Operator - expressions: _col1 (type: string), _col0 (type: string), (_col3 / _col4) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col2 (type: double) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + top n: 20 + Select Operator + expressions: _col1 (type: string), _col0 (type: string), (_col3 / _col4) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: double) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 750 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -668,167 +673,159 @@ POSTHOOK: Input: default@v_n13 #### A masked pattern was here #### STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string), value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), 2L (type: bigint), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col2), sum(_col3) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 2L (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - TableScan - Union - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col2), sum(_col3) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col2), sum(_col3) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((_col2 > 0L) and ((_col2 * 2L) = _col3)) (type: boolean) - Statistics: Num rows: 41 Data size: 7954 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: src - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col2 > 0L) and ((_col2 * 2L) = _col3)) (type: boolean) + Statistics: Num rows: 41 Data size: 7954 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), 1L (type: bigint), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 1L (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col2), sum(_col3) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Union 3 + Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/windowing_expressions.q.out ql/src/test/results/clientpositive/llap/windowing_expressions.q.out similarity index 53% rename from ql/src/test/results/clientpositive/windowing_expressions.q.out rename to ql/src/test/results/clientpositive/llap/windowing_expressions.q.out index 4ce99796d1..24d966b11b 100644 --- ql/src/test/results/clientpositive/windowing_expressions.q.out +++ ql/src/test/results/clientpositive/llap/windowing_expressions.q.out @@ -128,438 +128,438 @@ Manufacturer#5 1464.48 23 2 2482.58 2477.58 Manufacturer#5 1611.66 6 3 4094.24 4089.24 Manufacturer#5 1788.73 2 4 5882.969999999999 5877.969999999999 Manufacturer#5 1789.69 31 5 7672.66 7667.66 -PREHOOK: query: select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 limit 100 +PREHOOK: query: select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 order by s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -POSTHOOK: query: select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 limit 100 +POSTHOOK: query: select s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) from over10k_n22 order by s, si, f, si - lead(f, 3) over (partition by t order by bo,s,si,f desc) limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n22 #### A masked pattern was here #### alice allen 400 76.31 337.23 +alice allen 451 73.63 429.38 +alice allen 462 66.89 425.87 +alice allen 472 89.57 408.09 +alice allen 484 75.98 430.98 +alice allen 501 2.79 473.94 +alice allen 501 64.24 471.67 +alice allen 509 21.45 488.7 +alice brown 302 43.9 280.3 +alice brown 324 93.15 319.53 +alice brown 332 13.93 316.18 +alice brown 337 97.94 292.07 +alice brown 346 71.31 246.25 +alice brown 376 3.37 375.77 +alice brown 381 95.46 289.01 +alice brown 399 58.89 381.43 +alice brown 409 1.98 328.7 +alice brown 425 30.3 391.45 +alice brown 452 49.57 412.95 +alice brown 471 94.48 465.91 +alice brown 492 5.74 460.11 +alice brown 499 62.51 418.88 +alice carson 268 27.96 197.68 +alice carson 316 46.9 288.37 +alice carson 318 56.1 300.49 +alice carson 376 87.28 339.18 +alice carson 380 64.2 340.66 +alice carson 390 56.02 385.69 +alice carson 404 12.6 328.94 +alice carson 427 39.03 415.5 +alice carson 473 87.78 373.55 +alice carson 508 47.35 463.51 +alice davidson 270 60.39 202.25 +alice davidson 272 45.18 262.73 +alice davidson 287 90.86 187.29001 +alice davidson 298 53.84 242.49 +alice davidson 308 8.66 215.20999 +alice davidson 321 73.47 261.16 +alice davidson 328 70.35 231.20999 alice davidson 384 71.97 357.79 -alice king 455 2.48 395.93 -alice king 458 62.77 384.16998 -alice xylophone 485 26.21 464.05 -bob falkner 260 59.07 242.4 -bob ichabod 454 73.83 381.7 -bob polk 264 20.95 257.17 -bob underhill 454 17.6 424.94 -bob underhill 465 72.3 453.17 -bob van buren 433 6.83 398.4 -calvin ichabod 431 29.06 334.22 -david garcia 485 11.83 421.51 -ethan steinbeck 298 34.6 288.14 -fred ellison 376 96.78 330.76 -holly steinbeck 384 63.49 293.7 -holly underhill 318 9.86 269.91 -irene ellison 458 45.24 365.29 -irene underhill 307 90.3 244.19 -jessica johnson 494 48.09 490.18 -jessica king 459 92.71 452.2 -jessica white 284 62.81 209.08 -luke garcia 311 3.82 267.27 -luke young 451 6.8 429.0 -mike king 275 74.92 211.81 -oscar garcia 362 43.73 340.66 -priscilla laertes 316 22.0 296.06 -priscilla quirinius 423 63.19 362.72 -priscilla zipper 485 21.34 400.61 -quinn ellison 266 19.94 209.95 -quinn polk 507 60.28 447.66 -sarah robinson 320 84.39 309.74 -tom polk 346 56.05 320.33 -ulysses ellison 381 59.34 358.66 -ulysses quirinius 303 10.26 259.6 -ulysses robinson 313 25.67 269.31 -ulysses steinbeck 333 22.34 270.61 -victor allen 337 43.4 311.5 -victor hernandez 447 43.69 375.22 -victor xylophone 438 62.39 424.33 -wendy quirinius 279 25.5 250.25 -wendy robinson 275 71.78 262.88 -wendy xylophone 314 13.67 295.73 -xavier garcia 493 28.75 474.56 -zach thompson 386 12.12 377.63 -zach young 286 18.27 263.65 +alice davidson 402 78.24 364.38 +alice davidson 408 8.38 347.39 +alice davidson 408 71.2 331.46 +alice davidson 423 6.34 379.13 +alice davidson 431 60.34 375.19 +alice davidson 437 62.0 380.76 +alice davidson 445 92.22 406.65 +alice davidson 448 99.98 357.6 +alice davidson 479 24.54 431.76 +alice davidson 487 69.8 399.93 +alice ellison 256 38.16 166.31 +alice ellison 274 74.24 178.19 +alice ellison 296 91.53 208.67 +alice ellison 313 36.06 257.74 +alice ellison 320 54.25 309.71 +alice ellison 331 70.9 319.71 +alice ellison 335 92.23 312.1 +alice ellison 343 3.03 275.87 +alice ellison 354 68.23 320.59 +alice ellison 355 63.57 260.63 +alice ellison 374 75.1 310.54 +alice ellison 403 73.95 315.77 +alice ellison 405 23.05 399.6 +alice ellison 482 56.14 472.31 +alice ellison 490 77.51 473.66 alice falkner 280 18.44 227.7 -bob ellison 339 8.37 300.95 -bob johnson 374 22.35 326.49 -calvin white 280 52.3 198.32 -david carson 270 38.05 255.77 -david falkner 469 47.51 388.35 -david hernandez 408 81.68 339.27 -ethan underhill 339 14.23 256.26 -gabriella brown 498 80.65 413.25 -holly nixon 505 68.73 440.71 -holly polk 268 82.74 182.04001 -holly thompson 387 84.75 298.22 -irene young 458 64.29 401.8 -jessica miller 299 85.96 243.41 -katie ichabod 469 88.78 385.61 -luke ichabod 289 56.2 286.74 -luke king 337 55.59 274.88 -mike allen 465 83.39 383.03 -mike polk 500 2.26 427.74 -mike white 454 62.12 430.78 -mike xylophone 448 81.97 447.17 -nick nixon 335 72.26 240.78 -nick robinson 350 23.22 294.59 -oscar davidson 432 0.83 420.93 -oscar johnson 315 94.22 233.05 -oscar johnson 469 55.41 468.44 -oscar miller 324 11.07 265.19 -rachel davidson 507 81.95 468.78 -rachel thompson 344 0.56 246.12 -sarah miller 386 58.81 304.36 -sarah xylophone 275 38.22 177.48999 -sarah zipper 376 97.88 294.61 -tom hernandez 467 81.64 459.9 -tom hernandez 477 97.51 415.19 -tom steinbeck 414 81.39 361.87 -ulysses carson 343 7.1 314.22 -victor robinson 415 61.81 349.5 -victor thompson 344 52.13 NULL -xavier ovid 280 28.78 NULL -yuri xylophone 430 65.5 NULL -alice underhill 389 26.68 368.06 -alice underhill 446 6.49 444.21 -bob ovid 331 67.12 236.43 -bob van buren 406 20.94 383.32 -david falkner 406 1.79 374.34 -david miller 450 94.57 380.13 -ethan allen 380 22.68 375.6 -ethan king 395 31.66 361.51 -ethan nixon 475 69.87 431.39 -ethan polk 283 4.4 243.82 -fred allen 331 33.49 281.68 -fred king 511 43.61 457.22 -fred polk 261 39.18 248.73 -fred young 303 49.32 221.51001 -PREHOOK: query: select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 limit 100 +alice falkner 311 97.31 246.45 +alice falkner 323 90.25 246.19 +alice falkner 339 57.14 286.88 +alice falkner 342 90.76 336.73 +alice falkner 345 44.77 258.58002 +alice falkner 371 17.6 316.43 +alice falkner 382 19.32 366.19 +alice falkner 382 51.87 334.19 +alice falkner 389 32.7 334.83002 +alice falkner 393 7.23 293.11 +alice falkner 393 85.1 377.27 +alice falkner 452 89.86 366.1 +alice falkner 455 32.16 449.22 +alice falkner 477 18.84 378.93 +alice falkner 481 18.45 453.02 +alice falkner 500 51.04 417.44 +alice garcia 263 95.11 231.76 +alice garcia 299 15.56 274.92 +alice garcia 309 18.88 300.05 +alice garcia 325 52.15 297.22 +alice garcia 331 46.88 257.52 +alice garcia 366 11.33 269.1 +alice garcia 379 17.78 378.11 +alice garcia 388 51.6 363.71 +alice garcia 427 11.13 333.26 +alice garcia 446 38.83 392.47 +alice garcia 446 48.45 442.29 +alice garcia 459 1.29 445.45 +alice garcia 486 97.12 446.29 +alice hernandez 270 36.09 176.53 +alice hernandez 290 47.23 250.39 +alice hernandez 296 88.54 217.41 +alice hernandez 320 48.24 261.37 +alice hernandez 323 44.93 306.09 +PREHOOK: query: select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 order by s, i, i - lead(i, 3, 0) over (partition by si order by i,s)limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -POSTHOOK: query: select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 limit 100 +POSTHOOK: query: select s, i, i - lead(i, 3, 0) over (partition by si order by i,s) from over10k_n22 order by s, i, i - lead(i, 3, 0) over (partition by si order by i,s)limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -wendy garcia 65540 -18 -ethan thompson 65543 -20 -zach nixon 65549 -31 -alice robinson 65558 -28 -wendy nixon 65563 -33 -victor robinson 65580 -19 -ethan falkner 65586 -18 -victor davidson 65596 -17 -xavier quirinius 65599 -14 -fred quirinius 65604 -11 -nick zipper 65613 -3 -xavier van buren 65613 -7 -victor johnson 65615 -12 -alice ovid 65616 -24 -xavier ovid 65620 -23 -ulysses white 65627 -24 -sarah white 65640 -13 -calvin young 65643 -25 -victor thompson 65651 -42 -calvin johnson 65653 -53 -irene polk 65668 -45 -zach underhill 65693 -38 -quinn hernandez 65706 -27 -rachel ovid 65713 -24 -gabriella falkner 65731 -7 -zach white 65733 -8 -fred hernandez 65737 -7 -rachel ellison 65738 -6 -oscar steinbeck 65741 -6 +alice allen 65545 -15 +alice allen 65557 -38 +alice allen 65600 -29 +alice allen 65609 -5 +alice allen 65662 -18 +alice allen 65670 -15 +alice allen 65720 -15 +alice allen 65758 -18 +alice brown 65569 -13 +alice brown 65570 -45 +alice brown 65666 -22 +alice brown 65667 -7 +alice brown 65673 -8 +alice brown 65696 -16 +alice brown 65704 -6 +alice brown 65707 -16 +alice brown 65708 -15 +alice brown 65711 -10 +alice brown 65733 -22 +alice brown 65779 -10 +alice brown 65781 65781 +alice brown 65790 65790 +alice carson 65545 -44 +alice carson 65559 -56 +alice carson 65559 -15 +alice carson 65565 -22 +alice carson 65576 -26 +alice carson 65695 -30 +alice carson 65710 -6 +alice carson 65713 -27 +alice carson 65747 -6 +alice carson 65785 65785 +alice davidson 65544 -14 +alice davidson 65547 -41 +alice davidson 65554 -26 +alice davidson 65560 -4 +alice davidson 65563 -60 +alice davidson 65590 -22 +alice davidson 65596 -15 +alice davidson 65631 -18 +alice davidson 65641 -9 +alice davidson 65676 -20 +alice davidson 65677 -37 +alice davidson 65677 -23 +alice davidson 65690 -31 +alice davidson 65707 -29 +alice davidson 65740 -11 +alice davidson 65742 -42 +alice davidson 65747 -29 +alice davidson 65791 65791 +alice ellison 65537 -10 +alice ellison 65544 -3 +alice ellison 65557 -8 +alice ellison 65572 -19 +alice ellison 65612 -23 +alice ellison 65677 -37 +alice ellison 65681 -9 +alice ellison 65698 -16 +alice ellison 65699 -12 +alice ellison 65713 -6 +alice ellison 65730 -6 +alice ellison 65741 -35 alice ellison 65744 -8 -tom allen 65744 -19 -quinn quirinius 65747 -31 -victor hernandez 65752 -26 -holly xylophone 65763 -26 -david davidson 65778 65778 -ulysses young 65778 65778 -sarah brown 65789 65789 -xavier brown 65541 -16 -zach hernandez 65542 -18 -katie ichabod 65547 -19 -oscar young 65557 -15 -holly white 65560 -14 -priscilla laertes 65566 -9 -ethan king 65572 -6 -zach hernandez 65574 -10 -oscar thompson 65575 -13 -victor xylophone 65578 -16 -gabriella ellison 65584 -26 -nick quirinius 65588 -22 -holly robinson 65594 -18 -alice xylophone 65610 -16 -yuri brown 65610 -21 -sarah hernandez 65612 -26 -katie garcia 65626 -28 -jessica laertes 65631 -23 -ethan underhill 65638 -17 -irene young 65654 -37 -priscilla thompson 65654 -40 -luke quirinius 65655 -44 -david brown 65691 -20 -luke falkner 65694 -18 -priscilla miller 65699 -20 -rachel robinson 65711 -9 -ethan polk 65712 -10 -wendy brown 65719 -13 -mike underhill 65720 -18 -zach underhill 65722 -26 -nick zipper 65732 -20 -fred brown 65738 -18 -ulysses young 65748 -23 -nick davidson 65752 -19 -fred zipper 65756 -15 -yuri nixon 65771 -10 -zach hernandez 65771 -19 -zach zipper 65771 65771 -alice underhill 65781 65781 -oscar laertes 65790 65790 -sarah zipper 65546 -19 -bob falkner 65551 -17 -luke ovid 65551 -17 -katie allen 65565 -4 -nick falkner 65568 -5 -zach steinbeck 65568 -11 -oscar van buren 65569 -13 -gabriella young 65573 -11 -jessica ichabod 65579 -24 -david garcia 65582 -24 -nick xylophone 65584 -27 -calvin johnson 65603 -14 -xavier zipper 65606 -50 -alice nixon 65611 -58 -jessica laertes 65617 -62 -fred king 65656 -61 -priscilla underhill 65669 -48 -priscilla zipper 65679 -45 -nick king 65717 -11 -sarah polk 65717 -17 -irene quirinius 65724 -28 -tom laertes 65728 -25 -yuri johnson 65734 -27 -PREHOOK: query: select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 limit 100 +alice ellison 65745 -9 +alice ellison 65787 65787 +alice falkner 65596 -49 +alice falkner 65597 -13 +alice falkner 65611 -17 +alice falkner 65622 -11 +alice falkner 65669 -16 +alice falkner 65685 -14 +alice falkner 65690 -23 +alice falkner 65699 -9 +alice falkner 65709 -14 +alice falkner 65710 -19 +alice falkner 65715 -6 +alice falkner 65718 -20 +alice falkner 65722 -15 +alice falkner 65752 -34 +alice falkner 65773 -8 +alice falkner 65775 65775 +alice falkner 65785 65785 +alice garcia 65573 -5 +alice garcia 65613 -5 +alice garcia 65623 -16 +alice garcia 65630 -8 +alice garcia 65674 -30 +alice garcia 65675 -19 +alice garcia 65712 -17 +alice garcia 65725 -29 +alice garcia 65734 -12 +alice garcia 65744 -15 +alice garcia 65746 -15 +alice garcia 65746 -6 +alice garcia 65759 -9 +alice hernandez 65543 -19 +alice hernandez 65545 -8 +alice hernandez 65569 -25 +alice hernandez 65603 -6 +alice hernandez 65633 -30 +PREHOOK: query: select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 order by s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -POSTHOOK: query: select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 limit 100 +POSTHOOK: query: select s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) from over10k_n22 order by s, si, d, si - lag(d, 3) over (partition by b order by si,s,d) limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -jessica ellison 262 30.41 NULL -david young 266 45.12 NULL -jessica steinbeck 274 2.15 NULL -david zipper 275 43.45 244.59 -zach nixon 283 15.95 237.88 -holly allen 285 24.37 282.85 -irene garcia 292 33.54 248.55 -ulysses xylophone 292 44.66 276.05 -irene van buren 309 35.81 284.63 -sarah miller 312 6.65 278.46 -victor garcia 312 39.14 267.34000000000003 -ethan ichabod 319 29.4 283.19 -wendy falkner 322 10.02 315.35 -oscar miller 324 25.95 284.86 -david ovid 332 28.34 302.6 -alice zipper 333 3.38 322.98 -yuri nixon 333 8.28 307.05 -ulysses nixon 335 18.48 306.66 -david ovid 336 9.36 332.62 -calvin falkner 337 17.63 328.72 -katie quirinius 349 11.3 330.52 -quinn miller 351 22.46 341.64 -victor xylophone 357 38.58 339.37 -ethan garcia 368 9.2 356.7 -nick steinbeck 395 37.54 372.54 -ulysses ichabod 415 47.61 376.42 -rachel thompson 416 37.99 406.8 -calvin young 418 47.22 380.46 -katie xylophone 425 32.59 377.39 -nick quirinius 429 19.63 391.01 -ethan ellison 453 47.92 405.78 -irene nixon 454 48.03 421.40999999999997 -bob steinbeck 462 47.04 442.37 -luke robinson 462 47.48 414.08 -gabriella steinbeck 467 9.35 418.97 -tom hernandez 467 29.36 419.96 -irene polk 485 14.26 437.52 -mike xylophone 494 36.92 484.65 -calvin allen 499 39.99 469.64 -quinn steinbeck 503 16.62 488.74 -calvin thompson 263 30.87 NULL -rachel quirinius 263 29.46 NULL -ulysses garcia 263 31.85 NULL -mike steinbeck 266 48.57 235.13 -rachel young 275 14.75 245.54 -tom king 278 31.11 246.15 -oscar robinson 283 30.35 234.43 -zach allen 284 1.88 269.25 -bob king 308 27.61 276.89 -ulysses allen 310 22.77 279.65 -fred nixon 317 0.48 315.12 -gabriella robinson 321 0.33 293.39 -bob johnson 325 9.61 302.23 -rachel davidson 335 2.34 334.52 -fred brown 337 5.8 336.67 -wendy ellison 350 20.25 340.39 -zach falkner 391 13.67 388.66 -katie xylophone 410 39.09 404.2 -holly king 413 3.56 392.75 -sarah van buren 417 7.81 403.33 -calvin van buren 430 36.01 390.90999999999997 -katie white 434 33.56 430.44 -oscar quirinius 454 7.03 446.19 -zach young 505 18.19 468.99 -gabriella robinson 506 12.8 472.44 -sarah xylophone 507 16.09 499.97 -rachel thompson 267 46.87 NULL -gabriella van buren 271 41.04 NULL -mike steinbeck 284 11.44 NULL -ethan ovid 293 2.08 246.13 -luke falkner 293 40.67 251.96 -irene nixon 321 24.35 309.56 -mike van buren 327 2.58 324.92 -ulysses robinson 329 26.64 288.33 -quinn laertes 332 10.71 307.65 -tom polk 346 34.03 343.42 -jessica johnson 352 45.71 325.36 -xavier davidson 354 33.9 343.29 -wendy nixon 364 29.42 329.97 -jessica quirinius 375 47.33 329.29 -xavier brown 376 26.17 342.1 -gabriella davidson 383 18.87 353.58 -jessica brown 388 34.09 340.67 -gabriella garcia 391 32.44 364.83 -ethan miller 396 49.07 377.13 -bob garcia 416 7.82 381.90999999999997 -priscilla hernandez 416 29.94 383.56 -holly nixon 419 17.81 369.93 -nick underhill 429 39.54 421.18 -xavier falkner 434 0.88 404.06 -luke robinson 461 44.02 443.19 -bob underhill 465 22.58 425.46 -ulysses king 483 37.98 482.12 -jessica miller 486 26.14 441.98 -bob ovid 493 9.7 470.42 +alice allen 400 29.44 357.33 +alice allen 451 15.46 450.04 +alice allen 462 22.19 437.29 +alice allen 472 18.39 446.17 +alice allen 484 49.73 452.96 +alice allen 501 20.39 474.45 +alice allen 501 37.79 463.24 +alice allen 509 23.59 477.92 +alice brown 302 25.14 298.32 +alice brown 324 41.46 279.78 +alice brown 332 30.84 287.05 +alice brown 337 5.92 298.95 +alice brown 346 1.29 315.67 +alice brown 376 37.3 358.09 +alice brown 381 21.15 353.29 +alice brown 399 7.4 381.37 +alice brown 409 44.05 381.92 +alice brown 425 21.48 416.11 +alice brown 452 6.97 451.73 +alice brown 471 1.09 428.46 +alice brown 492 45.75 457.44 +alice brown 499 33.57 451.0 +alice carson 268 46.87 NULL +alice carson 316 5.41 268.85 +alice carson 318 46.5 282.3 +alice carson 376 1.57 327.3 +alice carson 380 32.67 371.19 +alice carson 390 43.55 373.81 +alice carson 404 13.47 371.33 +alice carson 427 42.02 400.78 +alice carson 473 12.26 445.88 +alice carson 508 6.26 498.74 +alice davidson 270 2.8 NULL +alice davidson 272 43.54 234.94 +alice davidson 287 1.94 241.16 +alice davidson 298 17.23 NULL +alice davidson 308 40.43 291.3 +alice davidson 321 21.01 286.47 +alice davidson 328 15.13 317.85 +alice davidson 384 31.52 374.19 +alice davidson 402 46.91 396.27 +alice davidson 408 10.74 389.95 +alice davidson 408 41.52 388.77 +alice davidson 423 13.02 378.39 +alice davidson 431 33.03 402.71 +alice davidson 437 47.25 405.83 +alice davidson 445 9.3 421.17 +alice davidson 448 40.87 446.87 +alice davidson 479 1.17 450.09 +alice davidson 487 8.8 477.18 +alice ellison 256 45.71 NULL +alice ellison 274 45.57 NULL +alice ellison 296 31.03 249.39 +alice ellison 313 43.73 264.84000000000003 +alice ellison 320 49.6 306.35 +alice ellison 331 29.9 305.08 +alice ellison 335 5.81 320.45 +alice ellison 343 27.11 335.73 +alice ellison 354 13.68 340.87 +alice ellison 355 35.92 336.11 +alice ellison 374 42.03 338.4 +alice ellison 403 12.56 396.39 +alice ellison 405 20.36 369.84000000000003 +alice ellison 482 13.5 473.05 +alice ellison 490 34.31 442.93 +alice falkner 280 49.8 NULL +alice falkner 311 22.2 297.53 +alice falkner 323 41.9 283.12 +alice falkner 339 48.11 304.3 +alice falkner 342 31.92 336.37 +alice falkner 345 4.55 328.34 +alice falkner 371 5.39 370.25 +alice falkner 382 5.2 334.86 +alice falkner 382 41.64 341.05 +alice falkner 389 30.11 342.25 +alice falkner 393 27.44 354.96 +alice falkner 393 41.29 345.05 +alice falkner 452 20.17 446.84 +alice falkner 455 49.76 411.42 +alice falkner 477 28.11 443.36 +alice falkner 481 23.51 446.49 alice falkner 500 37.85 462.02 -quinn xylophone 267 49.8 NULL -gabriella thompson 268 17.15 NULL -calvin xylophone 275 49.32 NULL -gabriella zipper 279 30.41 229.2 -PREHOOK: query: select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 limit 100 +alice garcia 263 9.55 NULL +alice garcia 299 42.31 256.34000000000003 +alice garcia 309 21.44 274.72 +alice garcia 325 19.93 307.14 +alice garcia 331 44.27 282.95 +alice garcia 366 1.29 334.93 +alice garcia 379 3.21 368.27 +alice garcia 388 23.49 364.37 +alice garcia 427 45.24 414.1 +alice garcia 446 7.18 413.61 +alice garcia 446 39.71 416.12 +alice garcia 459 6.36 437.26 +alice garcia 486 17.66 454.52 +alice hernandez 270 47.64 NULL +alice hernandez 290 41.78 262.16 +alice hernandez 296 49.58 266.28 +alice hernandez 320 41.45 278.38 +alice hernandez 323 42.75 319.19 +PREHOOK: query: select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 order by s, lag(s, 3, 'fred') over (partition by f order by b) limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -POSTHOOK: query: select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 limit 100 +POSTHOOK: query: select s, lag(s, 3, 'fred') over (partition by f order by b) from over10k_n22 order by s, lag(s, 3, 'fred') over (partition by f order by b) limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n22 #### A masked pattern was here #### -yuri thompson fred -bob ichabod fred -luke king fred -luke steinbeck fred -fred zipper fred -quinn miller fred -calvin van buren fred -holly steinbeck fred -david davidson fred -calvin thompson fred -calvin quirinius fred -david ovid fred -holly thompson fred -nick zipper fred -victor steinbeck fred -victor robinson fred -zach ovid fred -ulysses zipper fred -luke falkner fred -irene thompson fred -yuri johnson fred -ulysses falkner fred -gabriella robinson fred -alice robinson fred -priscilla xylophone fred -david laertes fred -mike underhill fred -victor van buren fred -holly falkner fred -priscilla falkner fred -ethan ovid fred -luke zipper fred -mike steinbeck fred -calvin white fred -alice quirinius fred -irene miller fred -wendy polk fred -nick young fred -yuri davidson fred -ethan ellison fred -zach hernandez fred -wendy miller fred -katie underhill fred -irene zipper fred -holly allen fred -quinn brown fred -calvin ovid fred -zach robinson fred -nick miller fred -mike allen fred -yuri van buren fred -priscilla young fred -zach miller fred -victor xylophone fred -sarah falkner fred -rachel ichabod fred -alice robinson fred -calvin ovid fred -calvin ovid fred -luke laertes fred -david hernandez fred -alice ovid fred -luke quirinius fred -oscar white fred -zach falkner fred -rachel thompson fred -priscilla king fred -xavier polk fred -wendy ichabod fred -rachel ovid fred -wendy allen fred -luke brown fred -mike brown fred -oscar ichabod fred -xavier garcia fred -yuri brown fred -bob xylophone fred -luke davidson fred -ethan quirinius fred -zach davidson fred -irene miller fred -wendy king fred -bob zipper fred -sarah thompson fred -bob carson fred -bob laertes fred -xavier allen fred -sarah robinson fred -david king fred -oscar davidson fred -victor hernandez fred -wendy polk fred -david ellison fred -ulysses johnson fred -jessica ovid fred -bob king fred -ulysses garcia fred -irene falkner fred -holly robinson fred -yuri white fred +alice allen fred +alice allen fred +alice allen fred +alice allen fred +alice allen fred +alice allen fred +alice allen fred +alice allen fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice brown fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice carson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice davidson fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice ellison fred +alice falkner alice underhill +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred +alice falkner fred underhill +alice garcia david garcia +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice garcia fred +alice hernandez fred +alice hernandez fred +alice hernandez fred +alice hernandez fred +alice hernandez fred PREHOOK: query: select p_mfgr, avg(p_retailprice) over(partition by p_mfgr, p_type order by p_mfgr) from part PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -571,28 +571,28 @@ POSTHOOK: Input: default@part Manufacturer#1 1753.76 Manufacturer#1 1632.66 Manufacturer#1 1602.59 -Manufacturer#1 1173.15 -Manufacturer#1 1173.15 Manufacturer#1 1414.42 Manufacturer#2 1800.7 Manufacturer#2 1690.68 Manufacturer#2 2031.98 -Manufacturer#2 1698.66 Manufacturer#2 1701.6 -Manufacturer#3 1922.98 Manufacturer#3 1410.39 Manufacturer#3 1671.68 Manufacturer#3 1190.27 Manufacturer#3 1337.29 -Manufacturer#4 1844.92 Manufacturer#4 1375.42 +Manufacturer#5 1788.73 +Manufacturer#1 1173.15 +Manufacturer#1 1173.15 +Manufacturer#2 1698.66 +Manufacturer#3 1922.98 +Manufacturer#4 1844.92 Manufacturer#4 1620.67 Manufacturer#4 1206.26 Manufacturer#4 1290.35 Manufacturer#5 1018.1 Manufacturer#5 1464.48 Manufacturer#5 1789.69 -Manufacturer#5 1788.73 Manufacturer#5 1611.66 PREHOOK: query: select p_mfgr, avg(p_retailprice) over(partition by p_mfgr order by p_type,p_mfgr rows between unbounded preceding and current row) from part PREHOOK: type: QUERY @@ -658,32 +658,33 @@ POSTHOOK: Lineage: t1_n142.a1 SCRIPT [(over10k_n22)over10k_n22.FieldSchema(name: POSTHOOK: Lineage: t1_n142.b1 SIMPLE [(over10k_n22)over10k_n22.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: t2_n83.a1 SCRIPT [(over10k_n22)over10k_n22.FieldSchema(name:t, type:tinyint, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:si, type:smallint, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:i, type:int, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:b, type:bigint, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:f, type:float, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:d, type:double, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:bo, type:boolean, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:s, type:string, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:ts, type:timestamp, comment:null), (over10k_n22)over10k_n22.FieldSchema(name:dec, type:decimal(4,2), comment:null), (over10k_n22)over10k_n22.FieldSchema(name:bin, type:binary, comment:null), ] POSTHOOK: Lineage: t2_n83.b1 SIMPLE [(over10k_n22)over10k_n22.FieldSchema(name:s, type:string, comment:null), ] -PREHOOK: query: select * from t1_n142 limit 3 +PREHOOK: query: select * from t1_n142 order by a1, b1 limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@t1_n142 #### A masked pattern was here #### -POSTHOOK: query: select * from t1_n142 limit 3 +POSTHOOK: query: select * from t1_n142 order by a1, b1 limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n142 #### A masked pattern was here #### -65542 rachel thompson -131088 oscar brown -262258 wendy steinbeck -PREHOOK: query: select * from t2_n83 limit 3 +65536 alice ichabod +65536 alice robinson +65536 bob robinson +PREHOOK: query: select * from t2_n83 order by a1, b1 limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@t2_n83 #### A masked pattern was here #### -POSTHOOK: query: select * from t2_n83 limit 3 +POSTHOOK: query: select * from t2_n83 order by a1, b1 limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@t2_n83 #### A masked pattern was here #### -65542 rachel thompson -131088 oscar brown -262258 wendy steinbeck +65536 alice ichabod +65536 alice robinson +65536 bob robinson PREHOOK: query: select p_mfgr, p_retailprice, p_size, round(sum(p_retailprice) over w1 , 2) + 50.0 = round(sum(lag(p_retailprice,1,50.0)) over w1 + (last_value(p_retailprice) over w1),2) from part window w1 as (distribute by p_mfgr sort by p_retailprice) +order by p_mfgr, p_retailprice, p_size limit 11 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -692,6 +693,7 @@ POSTHOOK: query: select p_mfgr, p_retailprice, p_size, round(sum(p_retailprice) over w1 , 2) + 50.0 = round(sum(lag(p_retailprice,1,50.0)) over w1 + (last_value(p_retailprice) over w1),2) from part window w1 as (distribute by p_mfgr sort by p_retailprice) +order by p_mfgr, p_retailprice, p_size limit 11 POSTHOOK: type: QUERY POSTHOOK: Input: default@part diff --git ql/src/test/results/clientpositive/llap/windowing_gby2.q.out ql/src/test/results/clientpositive/llap/windowing_gby2.q.out new file mode 100644 index 0000000000..b8494a0ef5 --- /dev/null +++ ql/src/test/results/clientpositive/llap/windowing_gby2.q.out @@ -0,0 +1,665 @@ +PREHOOK: query: explain +select rank() over (order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by ws.key +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: explain +select rank() over (order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by ws.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ws + Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), c_int (type: int) + outputColumnNames: key, c_int + Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(c_int) + keys: key (type: string) + minReductionHashAggr: 0.65 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col1 + Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), _col1 (type: bigint) + null sort order: az + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col1 + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: rank_window_0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select rank() over (order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by ws.key +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: select rank() over (order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by ws.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +1 +1 +1 +4 +4 +6 +7 +PREHOOK: query: explain +select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by cast(ws.key as int) +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: explain +select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by cast(ws.key as int) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ws + Statistics: Num rows: 20 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), c_int (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col1), sum(_col2) + keys: _col0 (type: int) + minReductionHashAggr: 0.7 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: bigint) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: string, _col2: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS LAST + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: avg_window_0 + arguments: _col0 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: avg_window_0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by cast(ws.key as int) +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank +from cbo_t3 ws +group by cast(ws.key as int) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +NULL +1.0 +2.0 +3.0 +PREHOOK: query: explain +select rank () over(partition by key order by sum(c_int - c_float) desc) , +dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), +percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) +from cbo_t3 +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: explain +select rank () over(partition by key order by sum(c_int - c_float) desc) , +dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), +percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) +from cbo_t3 +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cbo_t3 + Statistics: Num rows: 20 Data size: 3382 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), (UDFToFloat(c_int) - c_float) (type: float), (UDFToDouble(c_float) / UDFToDouble(c_int)) (type: double), c_int (type: int), ((UDFToDouble(c_float) / UDFToDouble(c_int)) - UDFToDouble(c_int)) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 20 Data size: 3382 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col2), sum(_col3), max(_col4), sum(_col5) + keys: _col0 (type: string), _col1 (type: string) + minReductionHashAggr: 0.100000024 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: double), _col3 (type: double), _col4 (type: int), _col5 (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), max(VALUE._col2), sum(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: double) + null sort order: az + sort order: +- + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col3 (type: double), _col4 (type: int), _col5 (type: double) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: int), VALUE._col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double, _col3: double, _col4: int, _col5: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 DESC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: rank_window_0 (type: int), _col1 (type: string), _col3 (type: double), _col4 (type: int), _col5 (type: double) + outputColumnNames: rank_window_0, _col1, _col3, _col4, _col5 + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: lower(_col1) (type: string), _col3 (type: double) + null sort order: az + sort order: ++ + Map-reduce partition columns: lower(_col1) (type: string) + Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: rank_window_0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: double) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col2 (type: string), KEY.reducesinkkey1 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col2, _col4, _col5, _col6 + Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col2: string, _col4: double, _col5: int, _col6: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS LAST + partition by: lower(_col2) + raw input shape: + window functions: + window function definition + alias: dense_rank_window_1 + arguments: _col4 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dense_rank_window_1 (type: int), _col0 (type: int), _col5 (type: int), _col6 (type: double) + outputColumnNames: dense_rank_window_1, _col0, _col5, _col6 + Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: int), _col6 (type: double) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col5 (type: int) + Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: dense_rank_window_1 (type: int), _col0 (type: int) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col6, _col7 + Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: int, _col6: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS LAST + partition by: _col6 + raw input shape: + window functions: + window function definition + alias: percent_rank_window_2 + arguments: _col7 + name: percent_rank + window function: GenericUDAFPercentRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: int), percent_rank_window_2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select rank () over(partition by key order by sum(c_int - c_float) desc) , +dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), +percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) +from cbo_t3 +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: select rank () over(partition by key order by sum(c_int - c_float) desc) , +dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), +percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) +from cbo_t3 +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +1 1 0.0 +1 1 0.0 +1 1 0.0 +1 1 0.0 +1 1 0.0 +1 1 0.0 +1 1 0.0 +PREHOOK: query: explain +select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank +from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 +group by ws.c_boolean +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: explain +select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank +from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 +group by ws.c_boolean +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ws + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 20 Data size: 1767 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), c_int (type: int), c_boolean (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: boolean) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: wr + filterExpr: cstring1 is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 899146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cstring1 is not null (type: boolean) + Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int), cstring1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: vectorized, llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 27 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3), sum(_col1) + keys: _col2 (type: boolean) + minReductionHashAggr: 0.9259259 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: boolean) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), (UDFToDouble(_col1) / UDFToDouble(_col2)) (type: double) + null sort order: az + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col1 (type: bigint), VALUE._col2 (type: bigint) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: bigint, _col2: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: (UDFToDouble(_col1) / UDFToDouble(_col2)) ASC NULLS LAST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: (UDFToDouble(_col1) / UDFToDouble(_col2)) + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: rank_window_0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank +from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 +group by ws.c_boolean +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### +POSTHOOK: query: select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank +from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 +group by ws.c_boolean +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Input: default@cbo_t3 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/windowing_multipartitioning.q.out ql/src/test/results/clientpositive/llap/windowing_multipartitioning.q.out similarity index 97% rename from ql/src/test/results/clientpositive/windowing_multipartitioning.q.out rename to ql/src/test/results/clientpositive/llap/windowing_multipartitioning.q.out index 7e8305cda0..51d4840773 100644 --- ql/src/test/results/clientpositive/windowing_multipartitioning.q.out +++ ql/src/test/results/clientpositive/llap/windowing_multipartitioning.q.out @@ -44,11 +44,11 @@ POSTHOOK: query: load data local inpath '../../data/files/over10k' into table ov POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@over10k_n11 -PREHOOK: query: select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 limit 100 +PREHOOK: query: select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 order by s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -POSTHOOK: query: select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 limit 100 +POSTHOOK: query: select s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) from over10k_n11 order by s, rank() over (partition by s order by si), sum(b) over (partition by s order by si) limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### @@ -61,6 +61,11 @@ alice allen 6 30064772191 alice allen 6 30064772191 alice allen 8 34359739722 alice brown 1 4294967391 +alice brown 10 42949674011 +alice brown 11 47244641313 +alice brown 12 51539608718 +alice brown 13 55834576122 +alice brown 14 60129543595 alice brown 2 8589934706 alice brown 3 12884902122 alice brown 4 17179869504 @@ -69,12 +74,8 @@ alice brown 6 25769804175 alice brown 7 30064771680 alice brown 8 34359739221 alice brown 9 38654706641 -alice brown 10 42949674011 -alice brown 11 47244641313 -alice brown 12 51539608718 -alice brown 13 55834576122 -alice brown 14 60129543595 alice carson 1 4294967446 +alice carson 10 42949673743 alice carson 2 8589934775 alice carson 3 12884902150 alice carson 4 17179869461 @@ -83,16 +84,7 @@ alice carson 6 25769804187 alice carson 7 30064771550 alice carson 8 34359738920 alice carson 9 38654706240 -alice carson 10 42949673743 alice davidson 1 4294967453 -alice davidson 2 8589934978 -alice davidson 3 12884902338 -alice davidson 4 17179869653 -alice davidson 5 21474836975 -alice davidson 6 25769804493 -alice davidson 7 30064772010 -alice davidson 8 34359739463 -alice davidson 9 38654706943 alice davidson 10 47244641824 alice davidson 10 47244641824 alice davidson 12 51539609264 @@ -102,7 +94,21 @@ alice davidson 15 64424511548 alice davidson 16 68719479029 alice davidson 17 73014446462 alice davidson 18 77309413954 +alice davidson 2 8589934978 +alice davidson 3 12884902338 +alice davidson 4 17179869653 +alice davidson 5 21474836975 +alice davidson 6 25769804493 +alice davidson 7 30064772010 +alice davidson 8 34359739463 +alice davidson 9 38654706943 alice ellison 1 4294967496 +alice ellison 10 42949674421 +alice ellison 11 47244641904 +alice ellison 12 51539609208 +alice ellison 13 55834576596 +alice ellison 14 60129544054 +alice ellison 15 64424511508 alice ellison 2 8589934942 alice ellison 3 12884902454 alice ellison 4 17179869870 @@ -111,21 +117,7 @@ alice ellison 6 25769804587 alice ellison 7 30064772066 alice ellison 8 34359739616 alice ellison 9 38654706933 -alice ellison 10 42949674421 -alice ellison 11 47244641904 -alice ellison 12 51539609208 -alice ellison 13 55834576596 -alice ellison 14 60129544054 -alice ellison 15 64424511508 alice falkner 1 4294967377 -alice falkner 2 8589934805 -alice falkner 3 12884902121 -alice falkner 4 17179869431 -alice falkner 5 21474836879 -alice falkner 6 25769804283 -alice falkner 7 30064771719 -alice falkner 8 38654706491 -alice falkner 8 38654706491 alice falkner 10 42949673903 alice falkner 11 51539608896 alice falkner 11 51539608896 @@ -134,7 +126,19 @@ alice falkner 14 60129543752 alice falkner 15 64424511125 alice falkner 16 68719478658 alice falkner 17 73014445956 +alice falkner 2 8589934805 +alice falkner 3 12884902121 +alice falkner 4 17179869431 +alice falkner 5 21474836879 +alice falkner 6 25769804283 +alice falkner 7 30064771719 +alice falkner 8 38654706491 +alice falkner 8 38654706491 alice garcia 1 4294967303 +alice garcia 10 47244641402 +alice garcia 10 47244641402 +alice garcia 12 51539608899 +alice garcia 13 55834576425 alice garcia 2 8589934839 alice garcia 3 12884902276 alice garcia 4 17179869705 @@ -143,10 +147,6 @@ alice garcia 6 25769804353 alice garcia 7 30064771681 alice garcia 8 34359739213 alice garcia 9 38654706564 -alice garcia 10 47244641402 -alice garcia 10 47244641402 -alice garcia 12 51539608899 -alice garcia 13 55834576425 alice hernandez 1 4294967345 alice hernandez 2 8589934782 alice hernandez 3 12884902197 @@ -168,36 +168,36 @@ where s = 'tom allen' or s = 'bob steinbeck' POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -bob steinbeck 11 4294967344 bob steinbeck 1 8589934849 -bob steinbeck 2 12884902321 -bob steinbeck 7 17179869870 -bob steinbeck 8 21474837212 -bob steinbeck 9 25769804712 -bob steinbeck 6 30064772008 bob steinbeck 10 34359739552 +bob steinbeck 11 4294967344 +bob steinbeck 2 12884902321 bob steinbeck 3 38654707094 bob steinbeck 4 42949674515 bob steinbeck 5 47244642041 -tom allen 9 4294967478 -tom allen 3 8589934816 -tom allen 7 12884902321 -tom allen 16 17179869673 -tom allen 8 21474837072 -tom allen 10 25769804454 -tom allen 15 30064771969 -tom allen 2 34359739365 -tom allen 6 38654706862 -tom allen 18 42949674383 +bob steinbeck 6 30064772008 +bob steinbeck 7 17179869870 +bob steinbeck 8 21474837212 +bob steinbeck 9 25769804712 tom allen 1 47244641842 -tom allen 5 51539609307 -tom allen 19 55834576824 -tom allen 17 60129544192 +tom allen 10 25769804454 tom allen 11 64424511531 -tom allen 4 68719478972 tom allen 12 73014446496 tom allen 13 77309413835 tom allen 14 81604381169 +tom allen 15 30064771969 +tom allen 16 17179869673 +tom allen 17 60129544192 +tom allen 18 42949674383 +tom allen 19 55834576824 +tom allen 2 34359739365 +tom allen 3 8589934816 +tom allen 4 68719478972 +tom allen 5 51539609307 +tom allen 6 38654706862 +tom allen 7 12884902321 +tom allen 8 21474837072 +tom allen 9 4294967478 PREHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition by si) from over10k_n11 where s = 'tom allen' or s = 'bob steinbeck' PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n11 @@ -206,36 +206,36 @@ POSTHOOK: query: select s, sum(i) over (partition by s), sum(f) over (partition POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -tom allen 1248023 47.16999816894531 -tom allen 1248023 77.77999877929688 -tom allen 1248023 11.069999694824219 -bob steinbeck 722083 83.52999877929688 -bob steinbeck 722083 38.33000183105469 -bob steinbeck 722083 28.479999542236328 -tom allen 1248023 2.8499999046325684 -tom allen 1248023 19.459999084472656 +bob steinbeck 722083 132.82000350952148 bob steinbeck 722083 26.290000915527344 -tom allen 1248023 26.239999771118164 +bob steinbeck 722083 28.479999542236328 bob steinbeck 722083 36.209999084472656 -tom allen 1248023 95.41000366210938 -tom allen 1248023 89.88999938964844 -tom allen 1248023 14.510000228881836 -bob steinbeck 722083 83.52999877929688 -tom allen 1248023 38.93000030517578 -tom allen 1248023 83.47000122070312 -tom allen 1248023 81.8499984741211 +bob steinbeck 722083 38.33000183105469 bob steinbeck 722083 47.810001373291016 -tom allen 1248023 15.84000015258789 -tom allen 1248023 52.779998779296875 -tom allen 1248023 39.4900016784668 -bob steinbeck 722083 80.7300033569336 -tom allen 1248023 11.300000190734863 -tom allen 1248023 68.46999740600586 bob steinbeck 722083 68.46999740600586 +bob steinbeck 722083 80.7300033569336 +bob steinbeck 722083 83.52999877929688 +bob steinbeck 722083 83.52999877929688 bob steinbeck 722083 9.699999809265137 -tom allen 1248023 55.38999938964844 -bob steinbeck 722083 132.82000350952148 +tom allen 1248023 11.069999694824219 +tom allen 1248023 11.300000190734863 tom allen 1248023 132.82000350952148 +tom allen 1248023 14.510000228881836 +tom allen 1248023 15.84000015258789 +tom allen 1248023 19.459999084472656 +tom allen 1248023 2.8499999046325684 +tom allen 1248023 26.239999771118164 +tom allen 1248023 38.93000030517578 +tom allen 1248023 39.4900016784668 +tom allen 1248023 47.16999816894531 +tom allen 1248023 52.779998779296875 +tom allen 1248023 55.38999938964844 +tom allen 1248023 68.46999740600586 +tom allen 1248023 77.77999877929688 +tom allen 1248023 81.8499984741211 +tom allen 1248023 83.47000122070312 +tom allen 1248023 89.88999938964844 +tom allen 1248023 95.41000366210938 PREHOOK: query: select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k_n11 where s = 'tom allen' or s = 'bob steinbeck' PREHOOK: type: QUERY @@ -246,35 +246,35 @@ where s = 'tom allen' or s = 'bob steinbeck' POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -tom allen 1 1 -tom allen 1 1 -tom allen 7 1 bob steinbeck 1 1 +bob steinbeck 1 1 +bob steinbeck 1 1 +bob steinbeck 1 2 +bob steinbeck 5 1 +bob steinbeck 5 1 +bob steinbeck 5 1 +bob steinbeck 5 1 bob steinbeck 5 1 bob steinbeck 5 1 -tom allen 7 1 -tom allen 1 1 bob steinbeck 5 1 tom allen 1 1 -bob steinbeck 1 1 -tom allen 7 1 +tom allen 1 1 +tom allen 1 1 +tom allen 1 1 +tom allen 1 1 tom allen 1 1 tom allen 7 1 -bob steinbeck 5 1 tom allen 7 1 tom allen 7 1 tom allen 7 1 -bob steinbeck 5 1 tom allen 7 1 tom allen 7 1 tom allen 7 1 -bob steinbeck 5 1 tom allen 7 1 tom allen 7 1 -bob steinbeck 1 2 -bob steinbeck 5 1 -tom allen 1 1 -bob steinbeck 1 1 +tom allen 7 1 +tom allen 7 1 +tom allen 7 1 tom allen 7 2 PREHOOK: query: select s, sum(f) over (partition by i), row_number() over (order by f) from over10k_n11 where s = 'tom allen' or s = 'bob steinbeck' PREHOOK: type: QUERY @@ -284,34 +284,34 @@ POSTHOOK: query: select s, sum(f) over (partition by i), row_number() over (orde POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -tom allen 2.8499999046325684 1 +bob steinbeck 26.290000915527344 9 +bob steinbeck 27.959999084472656 10 +bob steinbeck 28.479999542236328 11 +bob steinbeck 36.209999084472656 12 +bob steinbeck 38.33000183105469 13 +bob steinbeck 47.810001373291016 18 +bob steinbeck 80.7300033569336 23 +bob steinbeck 82.19000244140625 25 +bob steinbeck 83.52999877929688 27 +bob steinbeck 83.52999877929688 28 bob steinbeck 9.699999809265137 2 tom allen 11.069999694824219 3 tom allen 11.300000190734863 4 -tom allen 54.00000190734863 5 tom allen 15.84000015258789 6 tom allen 19.459999084472656 7 +tom allen 2.8499999046325684 1 tom allen 26.239999771118164 8 -bob steinbeck 26.290000915527344 9 -bob steinbeck 27.959999084472656 10 -bob steinbeck 28.479999542236328 11 -bob steinbeck 36.209999084472656 12 -bob steinbeck 38.33000183105469 13 tom allen 38.93000030517578 14 -tom allen 54.00000190734863 15 tom allen 40.5099983215332 16 tom allen 47.16999816894531 17 -bob steinbeck 47.810001373291016 18 tom allen 50.630001068115234 19 tom allen 52.779998779296875 20 +tom allen 54.00000190734863 15 +tom allen 54.00000190734863 5 tom allen 55.38999938964844 21 tom allen 77.77999877929688 22 -bob steinbeck 80.7300033569336 23 tom allen 81.8499984741211 24 -bob steinbeck 82.19000244140625 25 tom allen 83.47000122070312 26 -bob steinbeck 83.52999877929688 27 -bob steinbeck 83.52999877929688 28 tom allen 89.88999938964844 29 tom allen 95.41000366210938 30 PREHOOK: query: select s, rank() over w1, @@ -334,33 +334,33 @@ w2 as (partition by si order by f) POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n11 #### A masked pattern was here #### -tom allen 14 1 -tom allen 17 1 -tom allen 7 1 bob steinbeck 1 1 +bob steinbeck 10 1 bob steinbeck 11 1 +bob steinbeck 2 2 +bob steinbeck 3 1 +bob steinbeck 4 1 +bob steinbeck 5 1 +bob steinbeck 6 1 bob steinbeck 7 1 +bob steinbeck 8 1 +bob steinbeck 9 1 +tom allen 1 1 +tom allen 10 1 +tom allen 11 1 tom allen 12 1 -tom allen 15 1 -bob steinbeck 10 1 tom allen 13 1 -bob steinbeck 5 1 -tom allen 11 1 +tom allen 14 1 +tom allen 15 1 +tom allen 16 1 +tom allen 17 1 +tom allen 18 1 +tom allen 19 1 tom allen 2 1 -tom allen 9 1 -bob steinbeck 8 1 tom allen 3 1 tom allen 4 1 -tom allen 8 1 -bob steinbeck 3 1 -tom allen 10 1 -tom allen 18 1 -tom allen 19 1 -bob steinbeck 6 1 tom allen 5 1 -bob steinbeck 9 1 tom allen 6 2 -bob steinbeck 4 1 -tom allen 16 1 -tom allen 1 1 -bob steinbeck 2 2 +tom allen 7 1 +tom allen 8 1 +tom allen 9 1 diff --git ql/src/test/results/clientpositive/windowing_navfn.q.out ql/src/test/results/clientpositive/llap/windowing_navfn.q.out similarity index 88% rename from ql/src/test/results/clientpositive/windowing_navfn.q.out rename to ql/src/test/results/clientpositive/llap/windowing_navfn.q.out index a130be93e7..1335605de2 100644 --- ql/src/test/results/clientpositive/windowing_navfn.q.out +++ ql/src/test/results/clientpositive/llap/windowing_navfn.q.out @@ -58,54 +58,63 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (key = '238') (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '238') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: 0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: 0 (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = '238') (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: 0 ASC NULLS FIRST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 0 ASC NULLS FIRST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: row_number_window_0 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: row_number_window_0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/windowing_range_multiorder.q.out ql/src/test/results/clientpositive/llap/windowing_range_multiorder.q.out similarity index 75% rename from ql/src/test/results/clientpositive/windowing_range_multiorder.q.out rename to ql/src/test/results/clientpositive/llap/windowing_range_multiorder.q.out index a512fc8983..526968dcd8 100644 --- ql/src/test/results/clientpositive/windowing_range_multiorder.q.out +++ ql/src/test/results/clientpositive/llap/windowing_range_multiorder.q.out @@ -44,443 +44,443 @@ POSTHOOK: query: load data local inpath '../../data/files/over10k' into table ov POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@over10k_n17 -PREHOOK: query: select first_value(t) over ( partition by si order by i, b ) from over10k_n17 limit 100 +PREHOOK: query: select first_value(t) over ( partition by si order by i, b ) fv from over10k_n17 order by fv limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select first_value(t) over ( partition by si order by i, b ) from over10k_n17 limit 100 +POSTHOOK: query: select first_value(t) over ( partition by si order by i, b ) fv from over10k_n17 order by fv limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -51 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -48 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -47 -PREHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k_n17 limit 100 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-2 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +PREHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) lv from over10k_n17 order by lv limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) from over10k_n17 limit 100 +POSTHOOK: query: select last_value(i) over (partition by si, bo order by i, f desc range current row) lv from over10k_n17 order by lv limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -65543 -65549 -65558 -65580 -65586 -65596 -65616 -65620 -65627 -65640 -65643 -65706 -65713 -65737 -65744 -65752 -65778 -65540 -65563 -65599 -65604 -65613 -65613 -65615 -65651 -65653 -65668 -65693 -65731 -65733 -65738 -65741 -65744 -65747 -65763 -65778 -65789 -65541 -65547 -65560 -65572 -65574 -65575 -65578 -65588 -65594 -65610 -65691 -65694 -65711 -65719 -65722 -65738 -65756 -65790 -65542 -65557 -65566 -65584 -65610 -65612 -65626 -65631 -65638 -65654 -65654 -65655 -65699 -65712 -65720 -65732 -65748 -65752 -65771 -65771 -65771 -65781 -65565 -65569 -65573 -65582 -65584 -65606 -65656 -65669 -65717 -65724 -65728 -65761 -65762 -65770 -65771 -65781 -65546 -65551 -65551 -65568 -65568 -65579 -65603 -PREHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +PREHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) rn from over10k_n17 order by rn limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +POSTHOOK: query: select row_number() over (partition by si, bo order by i, f desc range between unbounded preceding and unbounded following) rn from over10k_n17 order by rn limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### 1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -19 -20 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -19 -20 -21 -22 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -1 -2 -3 -4 -5 -6 -7 -PREHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k_n17 limit 100 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PREHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) from over10k_n17 limit 100 +POSTHOOK: query: select s, si, i, avg(i) over (partition by s range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### +alice allen 400 65557 65640.125 alice allen 451 65662 65640.125 alice allen 462 65545 65640.125 -alice allen 501 65720 65640.125 -alice allen 501 65670 65640.125 -alice allen 484 65600 65640.125 alice allen 472 65609 65640.125 +alice allen 484 65600 65640.125 +alice allen 501 65670 65640.125 +alice allen 501 65720 65640.125 alice allen 509 65758 65640.125 -alice allen 400 65557 65640.125 -alice brown 425 65570 65696.71428571429 -alice brown 376 65708 65696.71428571429 -alice brown 324 65569 65696.71428571429 alice brown 302 65711 65696.71428571429 +alice brown 324 65569 65696.71428571429 +alice brown 332 65781 65696.71428571429 +alice brown 337 65707 65696.71428571429 +alice brown 346 65696 65696.71428571429 +alice brown 376 65708 65696.71428571429 alice brown 381 65704 65696.71428571429 +alice brown 399 65779 65696.71428571429 +alice brown 409 65667 65696.71428571429 +alice brown 425 65570 65696.71428571429 alice brown 452 65666 65696.71428571429 -alice brown 346 65696 65696.71428571429 alice brown 471 65733 65696.71428571429 -alice brown 409 65667 65696.71428571429 -alice brown 399 65779 65696.71428571429 -alice brown 332 65781 65696.71428571429 -alice brown 337 65707 65696.71428571429 -alice brown 499 65790 65696.71428571429 alice brown 492 65673 65696.71428571429 -alice carson 404 65710 65645.4 +alice brown 499 65790 65696.71428571429 +alice carson 268 65713 65645.4 +alice carson 316 65559 65645.4 +alice carson 318 65695 65645.4 alice carson 376 65576 65645.4 -alice carson 508 65545 65645.4 +alice carson 380 65785 65645.4 +alice carson 390 65747 65645.4 +alice carson 404 65710 65645.4 alice carson 427 65559 65645.4 alice carson 473 65565 65645.4 -alice carson 390 65747 65645.4 -alice carson 318 65695 65645.4 -alice carson 316 65559 65645.4 -alice carson 268 65713 65645.4 -alice carson 380 65785 65645.4 +alice carson 508 65545 65645.4 +alice davidson 270 65563 65648.5 +alice davidson 272 65742 65648.5 +alice davidson 287 65747 65648.5 alice davidson 298 65554 65648.5 -alice davidson 479 65631 65648.5 -alice davidson 445 65590 65648.5 +alice davidson 308 65560 65648.5 +alice davidson 321 65677 65648.5 +alice davidson 328 65547 65648.5 alice davidson 384 65676 65648.5 +alice davidson 402 65544 65648.5 +alice davidson 408 65707 65648.5 alice davidson 408 65791 65648.5 -alice davidson 321 65677 65648.5 -alice davidson 448 65641 65648.5 alice davidson 423 65740 65648.5 -alice davidson 270 65563 65648.5 alice davidson 431 65677 65648.5 -alice davidson 487 65596 65648.5 -alice davidson 402 65544 65648.5 -alice davidson 272 65742 65648.5 -alice davidson 287 65747 65648.5 -alice davidson 328 65547 65648.5 alice davidson 437 65690 65648.5 -alice davidson 308 65560 65648.5 -alice davidson 408 65707 65648.5 -alice ellison 405 65713 65669.13333333333 -alice ellison 490 65572 65669.13333333333 -alice ellison 354 65698 65669.13333333333 -alice ellison 331 65557 65669.13333333333 -alice ellison 313 65612 65669.13333333333 +alice davidson 445 65590 65648.5 +alice davidson 448 65641 65648.5 +alice davidson 479 65631 65648.5 +alice davidson 487 65596 65648.5 +alice ellison 256 65744 65669.13333333333 +alice ellison 274 65537 65669.13333333333 alice ellison 296 65741 65669.13333333333 -alice ellison 403 65544 65669.13333333333 -alice ellison 482 65681 65669.13333333333 +alice ellison 313 65612 65669.13333333333 alice ellison 320 65745 65669.13333333333 -alice ellison 274 65537 65669.13333333333 -alice ellison 256 65744 65669.13333333333 -alice ellison 355 65699 65669.13333333333 -alice ellison 343 65787 65669.13333333333 +alice ellison 331 65557 65669.13333333333 alice ellison 335 65730 65669.13333333333 +alice ellison 343 65787 65669.13333333333 +alice ellison 354 65698 65669.13333333333 +alice ellison 355 65699 65669.13333333333 alice ellison 374 65677 65669.13333333333 -alice falkner 342 65752 65695.76470588235 +alice ellison 403 65544 65669.13333333333 +alice ellison 405 65713 65669.13333333333 +alice ellison 482 65681 65669.13333333333 +alice ellison 490 65572 65669.13333333333 alice falkner 280 65597 65695.76470588235 -alice falkner 393 65611 65695.76470588235 -alice falkner 389 65699 65695.76470588235 -alice falkner 345 65773 65695.76470588235 -alice falkner 500 65775 65695.76470588235 +alice falkner 311 65715 65695.76470588235 alice falkner 323 65669 65695.76470588235 -alice falkner 393 65685 65695.76470588235 alice falkner 339 65785 65695.76470588235 -alice falkner 382 65690 65695.76470588235 +alice falkner 342 65752 65695.76470588235 +alice falkner 345 65773 65695.76470588235 alice falkner 371 65710 65695.76470588235 -alice falkner 481 65709 65695.76470588235 -alice falkner 311 65715 65695.76470588235 -alice falkner 477 65722 65695.76470588235 alice falkner 382 65622 65695.76470588235 -alice falkner 455 65718 65695.76470588235 +alice falkner 382 65690 65695.76470588235 +alice falkner 389 65699 65695.76470588235 +alice falkner 393 65611 65695.76470588235 +alice falkner 393 65685 65695.76470588235 alice falkner 452 65596 65695.76470588235 -alice garcia 388 65675 65688.76923076923 -alice garcia 366 65744 65688.76923076923 -alice garcia 331 65734 65688.76923076923 +alice falkner 455 65718 65695.76470588235 +alice falkner 477 65722 65695.76470588235 +alice falkner 481 65709 65695.76470588235 +alice falkner 500 65775 65695.76470588235 +alice garcia 263 65630 65688.76923076923 alice garcia 299 65623 65688.76923076923 +alice garcia 309 65746 65688.76923076923 +alice garcia 325 65573 65688.76923076923 +alice garcia 331 65734 65688.76923076923 +alice garcia 366 65744 65688.76923076923 alice garcia 379 65746 65688.76923076923 -alice garcia 486 65725 65688.76923076923 +alice garcia 388 65675 65688.76923076923 alice garcia 427 65674 65688.76923076923 -alice garcia 263 65630 65688.76923076923 -alice garcia 459 65712 65688.76923076923 -alice garcia 446 65759 65688.76923076923 -alice garcia 325 65573 65688.76923076923 -alice garcia 309 65746 65688.76923076923 alice garcia 446 65613 65688.76923076923 -alice hernandez 396 65545 65678.38888888889 -alice hernandez 336 65786 65678.38888888889 -alice hernandez 324 65720 65678.38888888889 +alice garcia 446 65759 65688.76923076923 +alice garcia 459 65712 65688.76923076923 +alice garcia 486 65725 65688.76923076923 alice hernandez 270 65717 65678.38888888889 +alice hernandez 290 65685 65678.38888888889 +alice hernandez 296 65569 65678.38888888889 +alice hernandez 320 65700 65678.38888888889 alice hernandez 323 65727 65678.38888888889 -PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100 +PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100 +POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### @@ -584,11 +584,11 @@ alice hernandez 290 65685 65701.0 alice hernandez 296 65569 65657.0 alice hernandez 320 65700 65667.75 alice hernandez 323 65727 65679.6 -PREHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100 +PREHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) m from over10k_n17 order by s, si, i, m limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) from over10k_n17 limit 100 +POSTHOOK: query: select s, si, i, min(i) over (partition by s order by si, i range between unbounded preceding and current row) m from over10k_n17 order by s, si, i, m limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### @@ -692,11 +692,11 @@ alice hernandez 290 65685 65685 alice hernandez 296 65569 65569 alice hernandez 320 65700 65569 alice hernandez 323 65727 65569 -PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k_n17 limit 100 +PREHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) from over10k_n17 limit 100 +POSTHOOK: query: select s, si, i, avg(i) over (partition by s order by si, i desc range between unbounded preceding and current row) a from over10k_n17 order by s, si, i, a limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### @@ -705,8 +705,8 @@ alice allen 451 65662 65609.5 alice allen 462 65545 65588.0 alice allen 472 65609 65593.25 alice allen 484 65600 65594.6 -alice allen 501 65720 65615.5 alice allen 501 65670 65623.28571428571 +alice allen 501 65720 65615.5 alice allen 509 65758 65640.125 alice brown 302 65711 65711.0 alice brown 324 65569 65640.0 @@ -741,8 +741,8 @@ alice davidson 321 65677 65640.5 alice davidson 328 65547 65627.14285714286 alice davidson 384 65676 65633.25 alice davidson 402 65544 65623.33333333333 -alice davidson 408 65791 65640.1 alice davidson 408 65707 65646.18181818182 +alice davidson 408 65791 65640.1 alice davidson 423 65740 65654.0 alice davidson 431 65677 65655.76923076923 alice davidson 437 65690 65658.21428571429 @@ -772,11 +772,11 @@ alice falkner 339 65785 65691.5 alice falkner 342 65752 65703.6 alice falkner 345 65773 65715.16666666667 alice falkner 371 65710 65714.42857142857 -alice falkner 382 65690 65711.375 alice falkner 382 65622 65701.44444444444 +alice falkner 382 65690 65711.375 alice falkner 389 65699 65701.2 -alice falkner 393 65685 65699.72727272728 alice falkner 393 65611 65692.33333333333 +alice falkner 393 65685 65699.72727272728 alice falkner 452 65596 65684.92307692308 alice falkner 455 65718 65687.28571428571 alice falkner 477 65722 65689.6 @@ -791,8 +791,8 @@ alice garcia 366 65744 65675.0 alice garcia 379 65746 65685.14285714286 alice garcia 388 65675 65683.875 alice garcia 427 65674 65682.77777777778 -alice garcia 446 65759 65690.4 alice garcia 446 65613 65683.36363636363 +alice garcia 446 65759 65690.4 alice garcia 459 65712 65685.75 alice garcia 486 65725 65688.76923076923 alice hernandez 270 65717 65717.0 @@ -800,11 +800,11 @@ alice hernandez 290 65685 65701.0 alice hernandez 296 65569 65657.0 alice hernandez 320 65700 65667.75 alice hernandez 323 65727 65679.6 -PREHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k_n17 limit 100 +PREHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) m from over10k_n17 order by si, bo, i, f, m limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) from over10k_n17 limit 100 +POSTHOOK: query: select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range between unbounded preceding and current row) m from over10k_n17 order by si, bo, i, f, m limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### @@ -829,8 +829,8 @@ POSTHOOK: Input: default@over10k_n17 256 true 65563 94.87 65563 256 true 65599 89.55 65599 256 true 65604 40.97 65604 -256 true 65613 93.29 65613 256 true 65613 78.27 65613 +256 true 65613 93.29 65613 256 true 65615 20.66 65615 256 true 65651 90.32 65651 256 true 65653 8.1 65653 @@ -872,8 +872,8 @@ POSTHOOK: Input: default@over10k_n17 257 true 65626 23.18 65626 257 true 65631 51.61 65631 257 true 65638 95.35 65638 -257 true 65654 24.54 65654 257 true 65654 9.8 65654 +257 true 65654 24.54 65654 257 true 65655 40.42 65655 257 true 65699 15.36 65699 257 true 65712 90.44 65712 @@ -881,9 +881,9 @@ POSTHOOK: Input: default@over10k_n17 257 true 65732 96.85 65732 257 true 65748 32.52 65748 257 true 65752 49.35 65752 -257 true 65771 95.58 65771 -257 true 65771 53.89 65771 257 true 65771 48.5 65771 +257 true 65771 53.89 65771 +257 true 65771 95.58 65771 257 true 65781 17.33 65781 258 false 65565 98.19 65565 258 false 65569 66.81 65569 @@ -902,333 +902,333 @@ POSTHOOK: Input: default@over10k_n17 258 false 65771 52.63 65771 258 false 65781 1.92 65781 258 true 65546 91.19 65546 -258 true 65551 91.56 65551 258 true 65551 88.97 65551 -258 true 65568 81.41 65568 +258 true 65551 91.56 65551 258 true 65568 13.57 65568 +258 true 65568 81.41 65568 258 true 65579 47.52 65579 258 true 65603 2.61 65603 -PREHOOK: query: select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +PREHOOK: query: select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by bo, r limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +POSTHOOK: query: select bo, rank() over (partition by i order by bo nulls first, b nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by bo, r limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### false 1 -false 2 -false 3 -false 4 -false 5 -false 6 -false 7 -false 8 -false 9 -false 10 -false 11 -false 11 -false 13 -false 14 -false 15 -false 16 -false 17 -false 18 -false 19 -false 20 -false 20 -false 22 -true 23 -true 24 -true 25 -true 26 -true 27 -true 28 -true 29 -true 30 -true 31 -true 32 -true 33 -true 34 -true 35 -true 36 -true 37 -true 37 -true 39 -true 40 -true 41 -true 42 -true 43 -true 44 -true 45 -false 1 -false 2 -false 3 -false 4 -false 5 -false 5 -false 5 -false 8 -false 9 -false 10 -false 11 -false 12 -false 13 -false 14 -false 15 -false 16 -false 17 -true 18 -true 19 -true 20 -true 21 -true 22 -true 23 -true 24 -true 25 -true 26 -true 27 -true 27 -true 29 -true 30 -true 31 -true 32 -true 33 -true 34 -true 35 -false 1 -false 2 -false 3 -false 4 -false 4 -false 6 -false 7 -false 8 -false 9 -false 10 -false 11 -false 12 -false 13 -false 14 -false 15 -false 16 -false 17 -false 18 -true 19 -true 20 -PREHOOK: query: select CAST(s as CHAR(12)), rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +false 1 +PREHOOK: query: select CAST(s as CHAR(12)) s, rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select CAST(s as CHAR(12)), rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +POSTHOOK: query: select CAST(s as CHAR(12)) s, rank() over (partition by i order by CAST(s as CHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -alice ichabo 1 -alice robins 2 -bob robinson 3 -calvin thomp 4 -david johnso 5 -david laerte 6 -david nixon 7 -david nixon 7 -ethan johnso 9 -ethan ovid 10 -ethan underh 11 -fred miller 12 -fred miller 12 -gabriella ga 14 -gabriella un 15 -holly white 16 -irene johnso 17 -katie elliso 18 -luke allen 19 -mike quirini 20 -mike white 21 -nick davidso 22 -oscar allen 23 -oscar garcia 24 -oscar ichabo 25 -oscar ovid 26 -oscar steinb 27 -priscilla ga 28 -priscilla wh 29 -priscilla xy 30 -priscilla yo 31 -rachel brown 32 -rachel ichab 33 -rachel xylop 34 -sarah thomps 35 -sarah thomps 35 -tom johnson 37 -tom steinbec 38 -ulysses polk 39 -victor johns 40 -wendy polk 41 -xavier david 42 -yuri ellison 43 -zach allen 44 -zach hernand 45 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 2 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 2 +alice davids 2 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 alice elliso 1 -bob carson 2 -calvin brown 3 -david xyloph 4 -ethan white 5 -fred johnson 6 -fred van bur 7 -gabriella ic 8 -holly laerte 9 -holly quirin 10 -jessica hern 11 -katie robins 12 -katie thomps 13 -luke nixon 14 -mike garcia 15 -mike hernand 16 -nick carson 17 -nick davidso 18 -oscar carson 19 -oscar robins 20 -priscilla wh 21 -sarah falkne 22 -sarah ichabo 23 -ulysses falk 24 -victor xylop 25 -wendy garcia 26 -wendy van bu 27 -xavier under 28 -yuri garcia 29 -yuri quirini 30 -yuri white 31 -zach falkner 32 -zach ichabod 33 -zach nixon 34 -zach ovid 35 -alice ichabo 1 -alice king 2 -alice robins 3 -calvin allen 4 -gabriella jo 5 -gabriella ni 6 -holly falkne 7 -holly hernan 8 -holly thomps 9 -katie nixon 10 -luke brown 11 -luke davidso 12 -luke white 13 -mike brown 14 -nick quirini 15 -oscar white 16 -priscilla xy 17 -quinn garcia 18 -quinn laerte 19 -rachel young 20 -PREHOOK: query: select CAST(s as VARCHAR(12)), rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +alice elliso 1 +alice elliso 2 +alice elliso 2 +alice elliso 2 +alice elliso 3 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 2 +alice hernan 1 +alice hernan 1 +alice hernan 1 +alice hernan 1 +alice hernan 1 +PREHOOK: query: select CAST(s as VARCHAR(12)) s, rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -POSTHOOK: query: select CAST(s as VARCHAR(12)), rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) from over10k_n17 limit 100 +POSTHOOK: query: select CAST(s as VARCHAR(12)) s, rank() over (partition by i order by CAST(s as VARCHAR(12)) nulls last range between unbounded preceding and unbounded following) r from over10k_n17 order by s, r limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_n17 #### A masked pattern was here #### -alice ichabo 1 -alice robins 2 -bob robinson 3 -calvin thomp 4 -david johnso 5 -david laerte 6 -david nixon 7 -david nixon 7 -ethan johnso 9 -ethan ovid 10 -ethan underh 11 -fred miller 12 -fred miller 12 -gabriella ga 14 -gabriella un 15 -holly white 16 -irene johnso 17 -katie elliso 18 -luke allen 19 -mike quirini 20 -mike white 21 -nick davidso 22 -oscar allen 23 -oscar garcia 24 -oscar ichabo 25 -oscar ovid 26 -oscar steinb 27 -priscilla ga 28 -priscilla wh 29 -priscilla xy 30 -priscilla yo 31 -rachel brown 32 -rachel ichab 33 -rachel xylop 34 -sarah thomps 35 -sarah thomps 35 -tom johnson 37 -tom steinbec 38 -ulysses polk 39 -victor johns 40 -wendy polk 41 -xavier david 42 -yuri ellison 43 -zach allen 44 -zach hernand 45 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice allen 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice brown 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 1 +alice carson 2 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 1 +alice davids 2 +alice davids 2 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 +alice elliso 1 alice elliso 1 -bob carson 2 -calvin brown 3 -david xyloph 4 -ethan white 5 -fred johnson 6 -fred van bur 7 -gabriella ic 8 -holly laerte 9 -holly quirin 10 -jessica hern 11 -katie robins 12 -katie thomps 13 -luke nixon 14 -mike garcia 15 -mike hernand 16 -nick carson 17 -nick davidso 18 -oscar carson 19 -oscar robins 20 -priscilla wh 21 -sarah falkne 22 -sarah ichabo 23 -ulysses falk 24 -victor xylop 25 -wendy garcia 26 -wendy van bu 27 -xavier under 28 -yuri garcia 29 -yuri quirini 30 -yuri white 31 -zach falkner 32 -zach ichabod 33 -zach nixon 34 -zach ovid 35 -alice ichabo 1 -alice king 2 -alice robins 3 -calvin allen 4 -gabriella jo 5 -gabriella ni 6 -holly falkne 7 -holly hernan 8 -holly thomps 9 -katie nixon 10 -luke brown 11 -luke davidso 12 -luke white 13 -mike brown 14 -nick quirini 15 -oscar white 16 -priscilla xy 17 -quinn garcia 18 -quinn laerte 19 -rachel young 20 +alice elliso 2 +alice elliso 2 +alice elliso 2 +alice elliso 3 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 1 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice falkne 2 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 1 +alice garcia 2 +alice hernan 1 +alice hernan 1 +alice hernan 1 +alice hernan 1 +alice hernan 1 diff --git ql/src/test/results/clientpositive/windowing_streaming.q.out ql/src/test/results/clientpositive/llap/windowing_streaming.q.out similarity index 52% rename from ql/src/test/results/clientpositive/windowing_streaming.q.out rename to ql/src/test/results/clientpositive/llap/windowing_streaming.q.out index ec24ce4dfb..77df4698ed 100644 --- ql/src/test/results/clientpositive/windowing_streaming.q.out +++ ql/src/test/results/clientpositive/llap/windowing_streaming.q.out @@ -62,55 +62,64 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_name (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col1: string, _col2: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col1 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: string), rank_window_0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_name (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: string), rank_window_0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -138,59 +147,75 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_name (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.8 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col1: string, _col2: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col1 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (rank_window_0 < 4) (type: boolean) - Statistics: Num rows: 8 Data size: 3896 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: ++ + keys: p_mfgr (type: string), p_name (type: string) + null sort order: az + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE + top n: 4 + Reduce Output Operator + key expressions: p_mfgr (type: string), p_name (type: string) + null sort order: az + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5694 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.8 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Select Operator - expressions: _col2 (type: string), rank_window_0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS LAST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 26 Data size: 12662 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (rank_window_0 < 4) (type: boolean) + Statistics: Num rows: 8 Data size: 3896 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: string), rank_window_0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -344,58 +369,75 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: ctinyint (type: tinyint), cdouble (type: double) - null sort order: az - sort order: ++ - Map-reduce partition columns: ctinyint (type: tinyint) - Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.8 - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col5 - Statistics: Num rows: 12288 Data size: 3403280 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: tinyint, _col5: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col5 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col5 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 12288 Data size: 3403280 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (rank_window_0 < 5) (type: boolean) - Statistics: Num rows: 4096 Data size: 1121976 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: ++ + keys: ctinyint (type: tinyint), cdouble (type: double) + null sort order: az + Map-reduce partition columns: ctinyint (type: tinyint) + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + top n: 5 + Reduce Output Operator + key expressions: ctinyint (type: tinyint), cdouble (type: double) + null sort order: az + sort order: ++ + Map-reduce partition columns: ctinyint (type: tinyint) + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.8 + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Select Operator - expressions: _col0 (type: tinyint), _col5 (type: double), rank_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4096 Data size: 40632 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4096 Data size: 40632 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 12288 Data size: 3365908 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: tinyint, _col5: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 ASC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 12288 Data size: 3365908 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (rank_window_0 < 5) (type: boolean) + Statistics: Num rows: 4096 Data size: 1097740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: tinyint), _col5 (type: double), rank_window_0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4096 Data size: 16396 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4096 Data size: 16396 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/windowing_udaf.q.out ql/src/test/results/clientpositive/llap/windowing_udaf.q.out new file mode 100644 index 0000000000..e344077b59 --- /dev/null +++ ql/src/test/results/clientpositive/llap/windowing_udaf.q.out @@ -0,0 +1,600 @@ +PREHOOK: query: drop table over10k_n4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table over10k_n4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table over10k_n4( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal, + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over10k_n4 +POSTHOOK: query: create table over10k_n4( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal, + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over10k_n4 +PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k_n4 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over10k_n4 +POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k_n4 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over10k_n4 +PREHOOK: query: select s, min(i) over (partition by s) m from over10k_n4 order by s, m limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +POSTHOOK: query: select s, min(i) over (partition by s) m from over10k_n4 order by s, m limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice allen 65545 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice brown 65569 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice carson 65545 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice davidson 65544 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice ellison 65537 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice falkner 65596 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice garcia 65573 +alice hernandez 65543 +alice hernandez 65543 +alice hernandez 65543 +alice hernandez 65543 +alice hernandez 65543 +PREHOOK: query: select s, avg(f) over (partition by si order by s) a from over10k_n4 order by s, a limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +POSTHOOK: query: select s, avg(f) over (partition by si order by s) a from over10k_n4 order by s, a limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +alice allen 21.450000762939453 +alice allen 33.51499891281128 +alice allen 33.51499891281128 +alice allen 66.88999938964844 +alice allen 73.62999725341797 +alice allen 75.9800033569336 +alice allen 76.30999755859375 +alice allen 89.56999969482422 +alice brown 1.9800000190734863 +alice brown 3.369999885559082 +alice brown 5.739999771118164 +alice brown 13.930000305175781 +alice brown 30.299999237060547 +alice brown 43.900001525878906 +alice brown 49.56999969482422 +alice brown 58.88999938964844 +alice brown 62.5099983215332 +alice brown 71.30999755859375 +alice brown 93.1500015258789 +alice brown 94.4800033569336 +alice brown 95.45999908447266 +alice brown 97.94000244140625 +alice carson 12.600000381469727 +alice carson 27.959999084472656 +alice carson 39.029998779296875 +alice carson 45.32499933242798 +alice carson 46.900001525878906 +alice carson 47.349998474121094 +alice carson 56.02000045776367 +alice carson 56.099998474121094 +alice carson 64.19999694824219 +alice carson 87.77999877929688 +alice davidson 6.340000152587891 +alice davidson 8.65999984741211 +alice davidson 24.540000915527344 +alice davidson 39.78999853134155 +alice davidson 39.78999853134155 +alice davidson 45.18000030517578 +alice davidson 53.84000015258789 +alice davidson 60.34000015258789 +alice davidson 60.38999938964844 +alice davidson 62.0 +alice davidson 69.80000305175781 +alice davidson 70.3499984741211 +alice davidson 71.97000122070312 +alice davidson 73.47000122070312 +alice davidson 78.23999786376953 +alice davidson 90.86000061035156 +alice davidson 92.22000122070312 +alice davidson 99.9800033569336 +alice ellison 3.0299999713897705 +alice ellison 23.049999237060547 +alice ellison 36.060001373291016 +alice ellison 38.15999984741211 +alice ellison 54.25 +alice ellison 56.13999938964844 +alice ellison 63.56999969482422 +alice ellison 68.2300033569336 +alice ellison 70.9000015258789 +alice ellison 73.94999694824219 +alice ellison 74.23999786376953 +alice ellison 75.0999984741211 +alice ellison 77.51000213623047 +alice ellison 91.52999877929688 +alice ellison 92.2300033569336 +alice falkner 17.600000381469727 +alice falkner 18.440000534057617 +alice falkner 18.450000762939453 +alice falkner 18.84000015258789 +alice falkner 32.15999984741211 +alice falkner 32.70000076293945 +alice falkner 35.59499931335449 +alice falkner 35.59499931335449 +alice falkner 44.77000045776367 +alice falkner 46.16499924659729 +alice falkner 46.16499924659729 +alice falkner 51.040000915527344 +alice falkner 57.13999938964844 +alice falkner 69.71500015258789 +alice falkner 90.25 +alice falkner 90.76000213623047 +alice falkner 97.30999755859375 +alice garcia 1.2899999618530273 +alice garcia 11.329999923706055 +alice garcia 15.5600004196167 +alice garcia 17.780000686645508 +alice garcia 18.8799991607666 +alice garcia 25.079999446868896 +alice garcia 43.64000129699707 +alice garcia 43.64000129699707 +alice garcia 51.599998474121094 +alice garcia 52.150001525878906 +alice garcia 58.89000129699707 +alice garcia 95.11000061035156 +alice garcia 97.12000274658203 +alice hernandez 5.760000228881836 +alice hernandez 16.020000457763672 +alice hernandez 29.579999923706055 +alice hernandez 42.28999900817871 +alice hernandez 42.57999897003174 +PREHOOK: query: select s, avg(i) over (partition by t, b order by s) a from over10k_n4 order by s, a limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +POSTHOOK: query: select s, avg(i) over (partition by t, b order by s) a from over10k_n4 order by s, a limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +alice allen 65545.0 +alice allen 65557.0 +alice allen 65600.0 +alice allen 65609.0 +alice allen 65662.0 +alice allen 65670.0 +alice allen 65720.0 +alice allen 65758.0 +alice brown 65569.0 +alice brown 65570.0 +alice brown 65666.0 +alice brown 65667.0 +alice brown 65673.0 +alice brown 65696.0 +alice brown 65704.0 +alice brown 65707.0 +alice brown 65708.0 +alice brown 65711.0 +alice brown 65733.0 +alice brown 65779.0 +alice brown 65781.0 +alice brown 65790.0 +alice carson 65545.0 +alice carson 65559.0 +alice carson 65559.0 +alice carson 65565.0 +alice carson 65576.0 +alice carson 65695.0 +alice carson 65710.0 +alice carson 65713.0 +alice carson 65747.0 +alice carson 65785.0 +alice davidson 65544.0 +alice davidson 65547.0 +alice davidson 65554.0 +alice davidson 65560.0 +alice davidson 65563.0 +alice davidson 65590.0 +alice davidson 65596.0 +alice davidson 65631.0 +alice davidson 65641.0 +alice davidson 65676.0 +alice davidson 65677.0 +alice davidson 65677.0 +alice davidson 65690.0 +alice davidson 65707.0 +alice davidson 65740.0 +alice davidson 65742.0 +alice davidson 65747.0 +alice davidson 65791.0 +alice ellison 65537.0 +alice ellison 65544.0 +alice ellison 65557.0 +alice ellison 65572.0 +alice ellison 65612.0 +alice ellison 65677.0 +alice ellison 65681.0 +alice ellison 65698.0 +alice ellison 65699.0 +alice ellison 65713.0 +alice ellison 65730.0 +alice ellison 65741.0 +alice ellison 65744.0 +alice ellison 65745.0 +alice ellison 65787.0 +alice falkner 65596.0 +alice falkner 65597.0 +alice falkner 65611.0 +alice falkner 65622.0 +alice falkner 65669.0 +alice falkner 65685.0 +alice falkner 65690.0 +alice falkner 65699.0 +alice falkner 65709.0 +alice falkner 65710.0 +alice falkner 65715.0 +alice falkner 65718.0 +alice falkner 65722.0 +alice falkner 65752.0 +alice falkner 65773.0 +alice falkner 65775.0 +alice falkner 65785.0 +alice garcia 65573.0 +alice garcia 65613.0 +alice garcia 65623.0 +alice garcia 65630.0 +alice garcia 65674.0 +alice garcia 65675.0 +alice garcia 65712.0 +alice garcia 65725.0 +alice garcia 65734.0 +alice garcia 65744.0 +alice garcia 65746.0 +alice garcia 65746.0 +alice garcia 65759.0 +alice hernandez 65543.0 +alice hernandez 65545.0 +alice hernandez 65569.0 +alice hernandez 65603.0 +alice hernandez 65633.0 +PREHOOK: query: select max(i) over w m from over10k_n4 window w as (partition by f) order by m limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +POSTHOOK: query: select max(i) over w m from over10k_n4 window w as (partition by f) order by m limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65536 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65537 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65538 +65539 +65539 +65539 +65539 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65540 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65541 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65542 +65543 +65543 +65543 +65543 +65543 +65543 +65543 +PREHOOK: query: select s, avg(d) over (partition by t order by f) a from over10k_n4 order by s, a limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +POSTHOOK: query: select s, avg(d) over (partition by t order by f) a from over10k_n4 order by s, a limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k_n4 +#### A masked pattern was here #### +alice allen 20.579565217391302 +alice allen 23.349705882352943 +alice allen 24.678000000000004 +alice allen 25.389591836734677 +alice allen 27.962444444444447 +alice allen 28.006444444444437 +alice allen 30.302499999999995 +alice allen 31.700000000000003 +alice brown 21.293750000000003 +alice brown 21.46707692307692 +alice brown 23.952876712328763 +alice brown 24.202500000000004 +alice brown 24.234 +alice brown 24.425263157894733 +alice brown 24.49444444444445 +alice brown 25.258749999999996 +alice brown 25.529374999999998 +alice brown 25.63012987012987 +alice brown 26.472439024390237 +alice brown 27.100638297872322 +alice brown 28.227894736842114 +alice brown 44.05 +alice carson 22.345500000000005 +alice carson 24.326500000000006 +alice carson 25.226555555555564 +alice carson 25.888947368421054 +alice carson 26.372325581395355 +alice carson 27.384999999999994 +alice carson 28.077727272727273 +alice carson 28.166315789473682 +alice carson 29.507619047619045 +alice carson 32.073 +alice davidson 21.56061224489796 +alice davidson 21.625 +alice davidson 22.001052631578947 +alice davidson 23.074945054945047 +alice davidson 23.111538461538455 +alice davidson 23.27782608695653 +alice davidson 23.656984126984128 +alice davidson 24.0865625 +alice davidson 24.153333333333325 +alice davidson 24.50972222222222 +alice davidson 24.514761904761908 +alice davidson 24.891249999999992 +alice davidson 26.08603773584905 +alice davidson 26.59527777777778 +alice davidson 27.091372549019617 +alice davidson 27.163 +alice davidson 27.687500000000004 +alice davidson 28.655531914893622 +alice ellison 21.5972 +alice ellison 22.633442622950824 +alice ellison 22.777288135593214 +alice ellison 23.614909090909098 +alice ellison 24.031666666666666 +alice ellison 24.23236363636364 +alice ellison 25.414374999999996 +alice ellison 25.66529411764705 +alice ellison 25.81180555555555 +alice ellison 25.936527777777783 +alice ellison 25.945555555555543 +alice ellison 26.596727272727286 +alice ellison 26.871666666666666 +alice ellison 28.015151515151512 +alice ellison 31.14 +alice falkner 18.2505 +alice falkner 21.492727272727276 +alice falkner 22.251200000000004 +alice falkner 23.94578947368421 +alice falkner 24.45659574468085 +alice falkner 24.605937500000003 +alice falkner 24.75875 +alice falkner 24.81884615384616 +alice falkner 25.019436619718313 +alice falkner 25.978918918918925 +alice falkner 26.547000000000008 +alice falkner 26.620697674418604 +alice falkner 26.884000000000004 +alice falkner 27.02307692307693 +alice falkner 27.775000000000002 +alice falkner 28.922857142857143 +alice falkner 29.141014492753634 +alice garcia 8.685 +alice garcia 24.2375 +alice garcia 24.572352941176472 +alice garcia 24.673170731707316 +alice garcia 25.012045454545454 +alice garcia 25.18111111111111 +alice garcia 25.32999999999999 +alice garcia 25.456136363636368 +alice garcia 25.898285714285716 +alice garcia 25.955616438356166 +alice garcia 28.09714285714286 +alice garcia 28.243999999999996 +alice garcia 29.501935483870966 +alice hernandez 18.293333333333333 +alice hernandez 19.2875 +alice hernandez 21.496166666666664 +alice hernandez 22.30551020408164 +alice hernandez 23.183939393939397 +PREHOOK: query: select key, max(value) over + (order by key rows between 10 preceding and 20 following) m +from src1 where length(key) > 10 +order by key, m +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select key, max(value) over + (order by key rows between 10 preceding and 20 following) m +from src1 where length(key) > 10 +order by key, m +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/windowing_windowspec3.q.out ql/src/test/results/clientpositive/llap/windowing_windowspec3.q.out similarity index 100% rename from ql/src/test/results/clientpositive/windowing_windowspec3.q.out rename to ql/src/test/results/clientpositive/llap/windowing_windowspec3.q.out index 440c1e5c0b..ca7d4a3de1 100644 --- ql/src/test/results/clientpositive/windowing_windowspec3.q.out +++ ql/src/test/results/clientpositive/llap/windowing_windowspec3.q.out @@ -48,8 +48,6 @@ POSTHOOK: query: select hirets, salary, sum(salary) over (partition by hirets ra POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_n0 #### A masked pattern was here #### -NULL 1500.0 3000.0 -NULL 1500.0 3000.0 1980-12-17 00:00:00 800.0 800.0 1981-02-20 00:00:00 1600.0 1600.0 1981-02-22 00:00:00 1250.0 1250.0 @@ -64,6 +62,8 @@ NULL 1500.0 3000.0 1982-01-23 00:00:00 1300.0 1300.0 1982-12-09 00:00:00 3000.0 3000.0 1983-01-12 00:00:00 1100.0 1100.0 +NULL 1500.0 3000.0 +NULL 1500.0 3000.0 PREHOOK: query: select deptno, empno, hiredate, salary, sum(salary) over (partition by deptno order by hiredate range 90 preceding), sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 90 following), @@ -89,18 +89,18 @@ POSTHOOK: Input: default@emp_n0 10 7782 1981-06-09 2450.0 2450.0 2450.0 NULL NULL 9300.0 2450.0 10 7839 1981-11-17 5000.0 5000.0 6300.0 NULL 1300.0 4300.0 7450.0 10 7934 1982-01-23 1300.0 6300.0 6300.0 5000.0 NULL 3000.0 8750.0 -10 7988 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 10 7987 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 +10 7988 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 20 7369 1980-12-17 800.0 800.0 800.0 NULL NULL 10075.0 800.0 20 7566 1981-04-02 2975.0 2975.0 2975.0 NULL NULL 7100.0 3775.0 -20 7902 1981-12-03 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0 20 7788 1982-12-09 3000.0 3000.0 4100.0 NULL 1100.0 1100.0 9775.0 20 7876 1983-01-12 1100.0 4100.0 4100.0 3000.0 NULL NULL 10875.0 +20 7902 1981-12-03 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0 30 7499 1981-02-20 1600.0 1600.0 5700.0 NULL 2850.0 6550.0 2850.0 30 7521 1981-02-22 1250.0 2850.0 5700.0 NULL 2850.0 6550.0 2850.0 +30 7654 1981-09-28 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0 30 7698 1981-05-01 2850.0 5700.0 5700.0 2850.0 NULL 3700.0 5700.0 30 7844 1981-09-08 1500.0 1500.0 3700.0 NULL 2200.0 2200.0 7200.0 -30 7654 1981-09-28 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0 30 7900 1981-12-03 950.0 3700.0 3700.0 2750.0 NULL NULL 9400.0 PREHOOK: query: select deptno, empno, hirets, salary, sum(salary) over (partition by deptno order by hirets range 7776000 preceding), @@ -127,18 +127,18 @@ POSTHOOK: Input: default@emp_n0 10 7782 1981-06-09 00:00:00 2450.0 2450.0 2450.0 NULL NULL 9300.0 2450.0 10 7839 1981-11-17 00:00:00 5000.0 5000.0 6300.0 NULL 1300.0 4300.0 7450.0 10 7934 1982-01-23 00:00:00 1300.0 6300.0 6300.0 5000.0 NULL 3000.0 8750.0 -10 7988 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 10 7987 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 +10 7988 NULL 1500.0 3000.0 3000.0 3000.0 NULL NULL 11750.0 20 7369 1980-12-17 00:00:00 800.0 800.0 800.0 NULL NULL 10075.0 800.0 20 7566 1981-04-02 00:00:00 2975.0 2975.0 2975.0 NULL NULL 7100.0 3775.0 -20 7902 1981-12-03 00:00:00 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0 20 7788 1982-12-09 00:00:00 3000.0 3000.0 4100.0 NULL 1100.0 1100.0 9775.0 20 7876 1983-01-12 00:00:00 1100.0 4100.0 4100.0 3000.0 NULL NULL 10875.0 +20 7902 1981-12-03 00:00:00 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0 30 7499 1981-02-20 00:00:00 1600.0 1600.0 5700.0 NULL 2850.0 6550.0 2850.0 30 7521 1981-02-22 00:00:00 1250.0 2850.0 5700.0 NULL 2850.0 6550.0 2850.0 +30 7654 1981-09-28 00:00:00 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0 30 7698 1981-05-01 00:00:00 2850.0 5700.0 5700.0 2850.0 NULL 3700.0 5700.0 30 7844 1981-09-08 00:00:00 1500.0 1500.0 3700.0 NULL 2200.0 2200.0 7200.0 -30 7654 1981-09-28 00:00:00 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0 30 7900 1981-12-03 00:00:00 950.0 3700.0 3700.0 2750.0 NULL NULL 9400.0 PREHOOK: query: select deptno, empno, bonus, avg(bonus) over (partition by deptno order by bonus range 200 preceding), @@ -162,21 +162,21 @@ from emp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_n0 #### A masked pattern was here #### +10 7782 NULL NULL NULL NULL NULL NULL NULL +10 7839 NULL NULL NULL NULL NULL NULL NULL 10 7934 NULL NULL NULL NULL NULL NULL NULL 10 7987 NULL NULL NULL NULL NULL NULL NULL -10 7839 NULL NULL NULL NULL NULL NULL NULL 10 7988 NULL NULL NULL NULL NULL NULL NULL -10 7782 NULL NULL NULL NULL NULL NULL NULL +20 7369 NULL NULL NULL NULL NULL NULL NULL +20 7566 NULL NULL NULL NULL NULL NULL NULL 20 7788 NULL NULL NULL NULL NULL NULL NULL -20 7902 NULL NULL NULL NULL NULL NULL NULL 20 7876 NULL NULL NULL NULL NULL NULL NULL -20 7566 NULL NULL NULL NULL NULL NULL NULL -20 7369 NULL NULL NULL NULL NULL NULL NULL -30 7844 0.0 0.0 0.0 NULL NULL 733.3333333333334 0.0 +20 7902 NULL NULL NULL NULL NULL NULL NULL 30 7499 300.0 300.0 400.0 NULL 500.0 1400.0 266.6666666666667 30 7521 500.0 400.0 400.0 300.0 NULL 1400.0 266.6666666666667 30 7654 1400.0 1400.0 1400.0 NULL NULL NULL 550.0 30 7698 NULL NULL NULL NULL NULL NULL 550.0 +30 7844 0.0 0.0 0.0 NULL NULL 733.3333333333334 0.0 30 7900 NULL NULL NULL NULL NULL NULL 550.0 PREHOOK: query: select deptno, empno, stock, salary, avg(salary) over (partition by deptno order by stock range 200 preceding), @@ -201,18 +201,18 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_n0 #### A masked pattern was here #### 10 7782 50.00 2450.0 2450.0 1687.5 NULL 1500.0 5000.0 1687.5 +10 7839 NULL 5000.0 5000.0 5000.0 5000.0 NULL NULL 2350.0 10 7934 100.00 1300.0 1875.0 1687.5 NULL NULL 5000.0 1687.5 10 7987 150.50 1500.0 1750.0 1687.5 NULL NULL 5000.0 1687.5 10 7988 200.00 1500.0 1687.5 1687.5 2450.0 NULL 5000.0 1687.5 -10 7839 NULL 5000.0 5000.0 5000.0 5000.0 NULL NULL 2350.0 +20 7369 NULL 800.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 20 7566 100.00 2975.0 2975.0 2975.0 NULL NULL 1975.0 2975.0 20 7788 NULL 3000.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 -20 7902 NULL 3000.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 20 7876 NULL 1100.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 -20 7369 NULL 800.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 +20 7902 NULL 3000.0 1975.0 1975.0 1975.0 NULL NULL 2175.0 30 7499 200.50 1600.0 1600.0 1450.0 NULL NULL 1683.3333333333333 1450.0 -30 7844 300.00 1500.0 1550.0 1400.0 NULL 1250.0 1900.0 1400.0 30 7521 300.50 1250.0 1450.0 1400.0 NULL 1250.0 1900.0 1400.0 30 7654 500.00 1250.0 1333.3333333333333 1333.3333333333333 1375.0 NULL 1900.0 1400.0 30 7698 NULL 2850.0 1900.0 1900.0 1900.0 NULL NULL 1566.6666666666667 +30 7844 300.00 1500.0 1550.0 1400.0 NULL 1250.0 1900.0 1400.0 30 7900 NULL 950.0 1900.0 1900.0 1900.0 NULL NULL 1566.6666666666667 diff --git ql/src/test/results/clientpositive/temp_table_partition_boolexpr.q.out ql/src/test/results/clientpositive/temp_table_partition_boolexpr.q.out deleted file mode 100644 index d4af83b320..0000000000 --- ql/src/test/results/clientpositive/temp_table_partition_boolexpr.q.out +++ /dev/null @@ -1,289 +0,0 @@ -PREHOOK: query: create temporary table part_boolexpr_temp(key int, value string) partitioned by (dt int, ts string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@part_boolexpr_temp -POSTHOOK: query: create temporary table part_boolexpr_temp(key int, value string) partitioned by (dt int, ts string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@part_boolexpr_temp -PREHOOK: query: select count(*) from part_boolexpr_temp where key = 'abc' -PREHOOK: type: QUERY -PREHOOK: Input: default@part_boolexpr_temp -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from part_boolexpr_temp where key = 'abc' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part_boolexpr_temp -#### A masked pattern was here #### -0 -PREHOOK: query: select * from part_boolexpr_temp where dt = 'abc' -PREHOOK: type: QUERY -PREHOOK: Input: default@part_boolexpr_temp -#### A masked pattern was here #### -POSTHOOK: query: select * from part_boolexpr_temp where dt = 'abc' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part_boolexpr_temp -#### A masked pattern was here #### -PREHOOK: query: explain select count(1) from srcpart where true -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where true -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where false -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where false -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where true and hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where true and hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where true or hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where true or hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where false or hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where false or hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where false and hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where false and hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select count(1) from srcpart where INPUT__FILE__NAME is not null -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: explain select count(1) from srcpart where INPUT__FILE__NAME is not null -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: INPUT__FILE__NAME is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: INPUT__FILE__NAME is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 2000 Data size: 37248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git ql/src/test/results/clientpositive/timestamp.q.out ql/src/test/results/clientpositive/timestamp.q.out deleted file mode 100644 index 90a46f58f4..0000000000 --- ql/src/test/results/clientpositive/timestamp.q.out +++ /dev/null @@ -1,352 +0,0 @@ -PREHOOK: query: explain select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain extended select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain extended select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -OPTIMIZED SQL: SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `c` -FROM (SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `$f0` -FROM `default`.`src` -UNION ALL -SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `$f0` -FROM `default`.`src`) AS `t1` -GROUP BY TRUE -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: boolean) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - auto parallelism: false - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: boolean) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [$hdt$_0:$hdt$_0-subquery1:src, $hdt$_0:$hdt$_0-subquery2:src] - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0 - columns.types timestamp - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select cast('2011-01-01 01:01:01' as timestamp) as c from src union select cast('2011-01-01 01:01:01' as timestamp) as c from src limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -2011-01-01 01:01:01 -PREHOOK: query: explain select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 20000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: TIMESTAMP'2011-01-01 01:01:01.123' (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select cast('2011-01-01 01:01:01.123' as timestamp) as c from src union select cast('2011-01-01 01:01:01.123' as timestamp) as c from src limit 5 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -2011-01-01 01:01:01.123 diff --git ql/src/test/results/clientpositive/udaf_number_format.q.out ql/src/test/results/clientpositive/udaf_number_format.q.out deleted file mode 100644 index 822ea784ba..0000000000 --- ql/src/test/results/clientpositive/udaf_number_format.q.out +++ /dev/null @@ -1,85 +0,0 @@ -PREHOOK: query: EXPLAIN SELECT - sum('a'), - avg('a'), - variance('a'), - std('a') -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT - sum('a'), - avg('a'), - variance('a'), - std('a') -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum('a'), count(), sum(null), sum(null) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double), _col1 (type: bigint), _col2 (type: double), _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), (_col0 / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / _col1) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT - sum('a'), - avg('a'), - variance('a'), - std('a') -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT - sum('a'), - avg('a'), - variance('a'), - std('a') -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0.0 0.0 NULL NULL diff --git ql/src/test/results/clientpositive/udf1.q.out ql/src/test/results/clientpositive/udf1.q.out deleted file mode 100644 index 9647770bcd..0000000000 --- ql/src/test/results/clientpositive/udf1.q.out +++ /dev/null @@ -1,206 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, - c5 STRING, c6 STRING, c7 STRING, c8 STRING, - c9 STRING, c10 STRING, c11 STRING, c12 STRING, c13 STRING, - c14 STRING, c15 STRING, c16 STRING, c17 STRING, - c18 STRING, c19 STRING, c20 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n1 -POSTHOOK: query: CREATE TABLE dest1_n1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, - c5 STRING, c6 STRING, c7 STRING, c8 STRING, - c9 STRING, c10 STRING, c11 STRING, c12 STRING, c13 STRING, - c14 STRING, c15 STRING, c16 STRING, c17 STRING, - c18 STRING, c19 STRING, c20 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n1 -PREHOOK: query: EXPLAIN -FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', - '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', - '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', - REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), - REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), - REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' - WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n1 -POSTHOOK: query: EXPLAIN -FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', - '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', - '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', - REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), - REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), - REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' - WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'TRUE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'FALSE' (type: string), 'FALSE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'FALSE' (type: string), 'TRUE' (type: string), 'TRUE' (type: string), 'acc' (type: string), 'abc' (type: string), 'abb' (type: string), 'hive' (type: string), 'hadoop' (type: string), 'AaAbAcA' (type: string), 'FALSE' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: string) - outputColumnNames: c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20 - Statistics: Num rows: 250 Data size: 442000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll'), compute_stats(c6, 'hll'), compute_stats(c7, 'hll'), compute_stats(c8, 'hll'), compute_stats(c9, 'hll'), compute_stats(c10, 'hll'), compute_stats(c11, 'hll'), compute_stats(c12, 'hll'), compute_stats(c13, 'hll'), compute_stats(c14, 'hll'), compute_stats(c15, 'hll'), compute_stats(c16, 'hll'), compute_stats(c17, 'hll'), compute_stats(c18, 'hll'), compute_stats(c19, 'hll'), compute_stats(c20, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct), _col9 (type: struct), _col10 (type: struct), _col11 (type: struct), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: struct), _col16 (type: struct), _col17 (type: struct), _col18 (type: struct), _col19 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5), compute_stats(VALUE._col6), compute_stats(VALUE._col7), compute_stats(VALUE._col8), compute_stats(VALUE._col9), compute_stats(VALUE._col10), compute_stats(VALUE._col11), compute_stats(VALUE._col12), compute_stats(VALUE._col13), compute_stats(VALUE._col14), compute_stats(VALUE._col15), compute_stats(VALUE._col16), compute_stats(VALUE._col17), compute_stats(VALUE._col18), compute_stats(VALUE._col19) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 - Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8800 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n1 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20 - Column Types: string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string - Table: default.dest1_n1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n1 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n1 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', - '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', - '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', - REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), - REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), - REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' - WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n1 -POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', - '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', - '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', - REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), - REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive'), REGEXP_REPLACE('hadoopAAA','A.*',''), - REGEXP_REPLACE('abc', '', 'A'), 'abc' RLIKE '' - WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n1 -POSTHOOK: Lineage: dest1_n1.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c10 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c11 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c12 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c13 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c14 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c15 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c16 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c17 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c18 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c19 SIMPLE [] -POSTHOOK: Lineage: dest1_n1.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c20 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c5 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c6 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c7 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c8 EXPRESSION [] -POSTHOOK: Lineage: dest1_n1.c9 EXPRESSION [] -PREHOOK: query: SELECT dest1_n1.* FROM dest1_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n1.* FROM dest1_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n1 -#### A masked pattern was here #### -TRUE FALSE TRUE TRUE TRUE FALSE FALSE FALSE TRUE TRUE FALSE TRUE TRUE acc abc abb hive hadoop AaAbAcA FALSE diff --git ql/src/test/results/clientpositive/udf3.q.out ql/src/test/results/clientpositive/udf3.q.out deleted file mode 100644 index 18abd9560c..0000000000 --- ql/src/test/results/clientpositive/udf3.q.out +++ /dev/null @@ -1,127 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n104(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n104 -POSTHOOK: query: CREATE TABLE dest1_n104(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n104 -PREHOOK: query: EXPLAIN -FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), -min(CAST('' AS INT)), max(CAST('' AS INT)) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n104 -POSTHOOK: query: EXPLAIN -FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), -min(CAST('' AS INT)), max(CAST('' AS INT)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n104 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(null), sum(null), min(null), max(null) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: int), _col3 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1), min(VALUE._col2), max(VALUE._col3) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: CAST( _col0 AS STRING) (type: string), CAST( _col1 AS STRING) (type: string), CAST( (UDFToDouble(_col1) / _col0) AS STRING) (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n104 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: c1, c2, c3, c4, c5 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll'), compute_stats(c3, 'hll'), compute_stats(c4, 'hll'), compute_stats(c5, 'hll') - mode: complete - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n104 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, c2, c3, c4, c5 - Column Types: string, string, string, string, string - Table: default.dest1_n104 - -PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), -min(CAST('' AS INT)), max(CAST('' AS INT)) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n104 -POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n104 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), -min(CAST('' AS INT)), max(CAST('' AS INT)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n104 -POSTHOOK: Lineage: dest1_n104.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1_n104.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1_n104.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1_n104.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1_n104.c5 EXPRESSION [] -PREHOOK: query: SELECT dest1_n104.* FROM dest1_n104 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n104 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n104.* FROM dest1_n104 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n104 -#### A masked pattern was here #### -0 NULL NULL NULL NULL diff --git ql/src/test/results/clientpositive/udf8.q.out ql/src/test/results/clientpositive/udf8.q.out deleted file mode 100644 index 8e8ca424b4..0000000000 --- ql/src/test/results/clientpositive/udf8.q.out +++ /dev/null @@ -1,95 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n54(c1 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n54 -POSTHOOK: query: CREATE TABLE dest1_n54(c1 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n54 -PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '' WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n54 -POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '' WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n54 -POSTHOOK: Lineage: dest1_n54.c1 SIMPLE [] -PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '1' WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n54 -POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n54 SELECT '1' WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n54 -POSTHOOK: Lineage: dest1_n54.c1 SIMPLE [] -PREHOOK: query: EXPLAIN -SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n54 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n54 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dest1_n54 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: string) - outputColumnNames: c1 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(c1), count(c1) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double), _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (_col0 / _col1) (type: double), _col0 (type: double), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n54 -#### A masked pattern was here #### -POSTHOOK: query: SELECT avg(c1), sum(c1), count(c1) FROM dest1_n54 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n54 -#### A masked pattern was here #### -1.0 1.0 1 diff --git ql/src/test/results/clientpositive/udf_10_trims.q.out ql/src/test/results/clientpositive/udf_10_trims.q.out deleted file mode 100644 index 41eefa3e8c..0000000000 --- ql/src/test/results/clientpositive/udf_10_trims.q.out +++ /dev/null @@ -1,158 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n5(c1 STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n5 -POSTHOOK: query: CREATE TABLE dest1_n5(c1 STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n5 -PREHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE dest1_n5 -SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) -FROM src -WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n5 -POSTHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE dest1_n5 -SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) -FROM src -WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n5 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 86.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'abc' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n5 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: c1 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n5 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1 - Column Types: string - Table: default.dest1_n5 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n5 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n5 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT OVERWRITE TABLE dest1_n5 -SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) -FROM src -WHERE src.key = 86 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n5 -POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n5 -SELECT trim(trim(trim(trim(trim(trim(trim(trim(trim(trim( ' abc ')))))))))) -FROM src -WHERE src.key = 86 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n5 -POSTHOOK: Lineage: dest1_n5.c1 SIMPLE [] diff --git ql/src/test/results/clientpositive/udf_case_column_pruning.q.out ql/src/test/results/clientpositive/udf_case_column_pruning.q.out deleted file mode 100644 index 658cfa8361..0000000000 --- ql/src/test/results/clientpositive/udf_case_column_pruning.q.out +++ /dev/null @@ -1,153 +0,0 @@ -PREHOOK: query: EXPLAIN -SELECT CASE a.key - WHEN '1' THEN 2 - WHEN '3' THEN 4 - ELSE 5 - END as key -FROM src a JOIN src b -ON a.key = b.key -ORDER BY key LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT CASE a.key - WHEN '1' THEN 2 - WHEN '3' THEN 4 - ELSE 5 - END as key -FROM src a JOIN src b -ON a.key = b.key -ORDER BY key LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), CASE WHEN ((key = '1')) THEN (2) WHEN ((key = '3')) THEN (4) ELSE (5) END (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 791 Data size: 3164 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - ListSink - -PREHOOK: query: SELECT CASE a.key - WHEN '1' THEN 2 - WHEN '3' THEN 4 - ELSE 5 - END as key -FROM src a JOIN src b -ON a.key = b.key -ORDER BY key LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT CASE a.key - WHEN '1' THEN 2 - WHEN '3' THEN 4 - ELSE 5 - END as key -FROM src a JOIN src b -ON a.key = b.key -ORDER BY key LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -5 -5 -5 -5 -5 -5 -5 -5 -5 -5 diff --git ql/src/test/results/clientpositive/udf_explode.q.out ql/src/test/results/clientpositive/udf_explode.q.out deleted file mode 100644 index 0143f3160b..0000000000 --- ql/src/test/results/clientpositive/udf_explode.q.out +++ /dev/null @@ -1,447 +0,0 @@ -PREHOOK: query: DESCRIBE FUNCTION explode -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: DESCRIBE FUNCTION explode -POSTHOOK: type: DESCFUNCTION -explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns -PREHOOK: query: DESCRIBE FUNCTION EXTENDED explode -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: DESCRIBE FUNCTION EXTENDED explode -POSTHOOK: type: DESCFUNCTION -explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns -Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode -Function type:BUILTIN -PREHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: src - Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - ListSink - -PREHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Group By Operator - aggregations: count(1) - keys: col (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: int) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [a:src] - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types int:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 -2 -3 -PREHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 -2 -3 -PREHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src tablesample (1 rows)) a GROUP BY a.myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 1 -2 1 -3 1 -PREHOOK: query: EXPLAIN EXTENDED SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: src - Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: map(1:'one',2:'two',3:'three') (type: map) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - ListSink - -PREHOOK: query: EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: map(1:'one',2:'two',3:'three') (type: map) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Group By Operator - aggregations: count(1) - keys: key (type: int), value (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - numBuckets: -1 - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col2 (type: bigint) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [a:src] - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types int:string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 one -2 two -3 three -PREHOOK: query: SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT a.key, a.val, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) AS (key,val) FROM src tablesample (1 rows)) a GROUP BY a.key, a.val -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 one 1 -2 two 1 -3 three 1 -PREHOOK: query: drop table lazy_array_map -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table lazy_array_map -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table lazy_array_map (map_col map, array_col array) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lazy_array_map -POSTHOOK: query: create table lazy_array_map (map_col map, array_col array) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lazy_array_map -PREHOOK: query: INSERT OVERWRITE TABLE lazy_array_map select map(1,'one',2,'two',3,'three'), array('100','200','300') FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@lazy_array_map -POSTHOOK: query: INSERT OVERWRITE TABLE lazy_array_map select map(1,'one',2,'two',3,'three'), array('100','200','300') FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@lazy_array_map -POSTHOOK: Lineage: lazy_array_map.array_col EXPRESSION [] -POSTHOOK: Lineage: lazy_array_map.map_col EXPRESSION [] -PREHOOK: query: SELECT array_col, myCol from lazy_array_map lateral view explode(array_col) X AS myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@lazy_array_map -#### A masked pattern was here #### -POSTHOOK: query: SELECT array_col, myCol from lazy_array_map lateral view explode(array_col) X AS myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lazy_array_map -#### A masked pattern was here #### -["100","200","300"] 100 -["100","200","300"] 200 -["100","200","300"] 300 -PREHOOK: query: SELECT map_col, myKey, myValue from lazy_array_map lateral view explode(map_col) X AS myKey, myValue -PREHOOK: type: QUERY -PREHOOK: Input: default@lazy_array_map -#### A masked pattern was here #### -POSTHOOK: query: SELECT map_col, myKey, myValue from lazy_array_map lateral view explode(map_col) X AS myKey, myValue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lazy_array_map -#### A masked pattern was here #### -{1:"one",2:"two",3:"three"} 1 one -{1:"one",2:"two",3:"three"} 2 two -{1:"one",2:"two",3:"three"} 3 three diff --git ql/src/test/results/clientpositive/udf_folder_constants.q.out ql/src/test/results/clientpositive/udf_folder_constants.q.out deleted file mode 100644 index bada547758..0000000000 --- ql/src/test/results/clientpositive/udf_folder_constants.q.out +++ /dev/null @@ -1,134 +0,0 @@ -PREHOOK: query: drop table if exists udf_tb1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists udf_tb1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table if exists udf_tb2 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists udf_tb2 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table udf_tb1 (year int, month int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@udf_tb1 -POSTHOOK: query: create table udf_tb1 (year int, month int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@udf_tb1 -PREHOOK: query: create table udf_tb2(month int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@udf_tb2 -POSTHOOK: query: create table udf_tb2(month int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@udf_tb2 -PREHOOK: query: insert into udf_tb1 values(2001, 11) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@udf_tb1 -POSTHOOK: query: insert into udf_tb1 values(2001, 11) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@udf_tb1 -POSTHOOK: Lineage: udf_tb1.month SCRIPT [] -POSTHOOK: Lineage: udf_tb1.year SCRIPT [] -PREHOOK: query: insert into udf_tb2 values(11) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@udf_tb2 -POSTHOOK: query: insert into udf_tb2 values(11) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@udf_tb2 -POSTHOOK: Lineage: udf_tb2.month SCRIPT [] -PREHOOK: query: explain -select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) -PREHOOK: type: QUERY -PREHOOK: Input: default@udf_tb1 -PREHOOK: Input: default@udf_tb2 -#### A masked pattern was here #### -POSTHOOK: query: explain -select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@udf_tb1 -POSTHOOK: Input: default@udf_tb2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: udf_tb1 - filterExpr: ((year = 2001) and month is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((year = 2001) and month is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: month (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: b - filterExpr: month is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: month is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: month (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 978336000L (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) -PREHOOK: type: QUERY -PREHOOK: Input: default@udf_tb1 -PREHOOK: Input: default@udf_tb2 -#### A masked pattern was here #### -POSTHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@udf_tb1 -POSTHOOK: Input: default@udf_tb2 -#### A masked pattern was here #### -978336000 diff --git ql/src/test/results/clientpositive/udf_isops_simplify.q.out ql/src/test/results/clientpositive/udf_isops_simplify.q.out deleted file mode 100644 index ef721435a5..0000000000 --- ql/src/test/results/clientpositive/udf_isops_simplify.q.out +++ /dev/null @@ -1,368 +0,0 @@ -PREHOOK: query: create table t (a integer) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t -POSTHOOK: query: create table t (a integer) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t -PREHOOK: query: explain select not ((a>0) is not true) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is not true) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (_col0 > 0) is true (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select not ((a>0) is not false) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is not false) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (_col0 > 0) is false (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select not ((a>0) is not null) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is not null) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 is null (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select not ((a>0) is true) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is true) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (_col0 > 0) is not true (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select not ((a>0) is false) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is false) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (_col0 > 0) is not false (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select not ((a>0) is null) from t group by a -PREHOOK: type: QUERY -PREHOOK: Input: default@t -#### A masked pattern was here #### -POSTHOOK: query: explain select not ((a>0) is null) from t group by a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 is not null (type: boolean) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git ql/src/test/results/clientpositive/udtf_explode.q.out ql/src/test/results/clientpositive/udtf_explode.q.out deleted file mode 100644 index 1b941b87bb..0000000000 --- ql/src/test/results/clientpositive/udtf_explode.q.out +++ /dev/null @@ -1,607 +0,0 @@ -PREHOOK: query: DESCRIBE FUNCTION explode -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: DESCRIBE FUNCTION explode -POSTHOOK: type: DESCFUNCTION -explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns -PREHOOK: query: DESCRIBE FUNCTION EXTENDED explode -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: DESCRIBE FUNCTION EXTENDED explode -POSTHOOK: type: DESCFUNCTION -explode(a) - separates the elements of array a into multiple rows, or the elements of a map into multiple rows and columns -Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode -Function type:BUILTIN -PREHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 3 - Processor Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Select Operator - expressions: col (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - ListSink - -PREHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Select Operator - expressions: col (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - null sort order: - numBuckets: -1 - sort order: - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - TopN: 3 - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: int) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [$hdt$_0:$hdt$_0:$hdt$_0:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types int,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: int) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10004 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types int,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types int,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types int:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 -2 -3 -PREHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(array(1,2,3)) AS (myCol) FROM src LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 -2 -3 -PREHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT a.myCol, count(1) FROM (SELECT explode(array(1,2,3)) AS myCol FROM src LIMIT 3) a GROUP BY a.myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 1 -2 1 -3 1 -PREHOOK: query: EXPLAIN SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 3 - Processor Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: map(1:'one',2:'two',3:'three') (type: map) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - ListSink - -PREHOOK: query: EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: map(1:'one',2:'two',3:'three') (type: map) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - null sort order: - numBuckets: -1 - sort order: - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - TopN: 3 - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: int), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [$hdt$_0:$hdt$_0:$hdt$_0:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: int), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2 - columns.types int,string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - numBuckets: -1 - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col2 (type: bigint) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10004 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2 - columns.types int,string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2 - columns.types int,string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types int:string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 one -2 two -3 three -PREHOOK: query: SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT a.myKey, a.myVal, count(1) FROM (SELECT explode(map(1,'one',2,'two',3,'three')) as (myKey,myVal) FROM src LIMIT 3) a GROUP BY a.myKey, a.myVal -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1 one 1 -2 two 1 -3 three 1 -PREHOOK: query: SELECT src.key, myCol FROM src lateral view explode(array(1,2,3)) x AS myCol LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT src.key, myCol FROM src lateral view explode(array(1,2,3)) x AS myCol LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -238 1 -238 2 -238 3 -PREHOOK: query: SELECT src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -238 1 one -238 2 two -238 3 three -PREHOOK: query: SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT BLOCK__OFFSET__INSIDE__FILE, src.key, myKey, myVal FROM src lateral view explode(map(1,'one',2,'two',3,'three')) x AS myKey,myVal LIMIT 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 238 1 one -0 238 2 two -0 238 3 three diff --git ql/src/test/results/clientpositive/udtf_json_tuple.q.out ql/src/test/results/clientpositive/udtf_json_tuple.q.out deleted file mode 100644 index b2f1edcb42..0000000000 --- ql/src/test/results/clientpositive/udtf_json_tuple.q.out +++ /dev/null @@ -1,471 +0,0 @@ -PREHOOK: query: create table json_t (key string, jstring string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@json_t -POSTHOOK: query: create table json_t (key string, jstring string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@json_t -PREHOOK: query: insert overwrite table json_t -select * from ( - select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src tablesample (1 rows) - union all - select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src tablesample (1 rows) - union all - select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src tablesample (1 rows) - union all - select '4', cast(null as string) from src tablesample (1 rows) - union all - select '5', '{"f1": "", "f5": null}' from src tablesample (1 rows) - union all - select '6', '[invalid JSON string]' from src tablesample (1 rows) -) s -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@json_t -POSTHOOK: query: insert overwrite table json_t -select * from ( - select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src tablesample (1 rows) - union all - select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src tablesample (1 rows) - union all - select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src tablesample (1 rows) - union all - select '4', cast(null as string) from src tablesample (1 rows) - union all - select '5', '{"f1": "", "f5": null}' from src tablesample (1 rows) - union all - select '6', '[invalid JSON string]' from src tablesample (1 rows) -) s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@json_t -POSTHOOK: Lineage: json_t.jstring EXPRESSION [] -POSTHOOK: Lineage: json_t.key EXPRESSION [] -PREHOOK: query: explain -select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 3179 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Select Operator - expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - function name: json_tuple - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -1 value1 value2 3 NULL 5.23 -2 value12 2 value3 4.01 NULL -3 value13 2 value33 value44 5.01 -4 NULL NULL NULL NULL NULL -5 NULL NULL NULL NULL -6 NULL NULL NULL NULL NULL -PREHOOK: query: explain -select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE - function name: json_tuple - Select Operator - expressions: c0 (type: string), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - null sort order: zzz - sort order: +++ - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: string), _col4 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### - NULL NULL NULL NULL -value1 value2 3 NULL 5.23 -value12 2 value3 4.01 NULL -value13 2 value33 value44 5.01 -NULL NULL NULL NULL NULL -NULL NULL NULL NULL NULL -PREHOOK: query: explain -select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 3179 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col6 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string) - Select Operator - expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - function name: json_tuple - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 6364 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col6 (type: string), _col9 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -1 value2 5.23 -2 2 NULL -3 2 5.01 -4 NULL NULL -5 NULL NULL -6 NULL NULL -PREHOOK: query: explain -select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 6 Data size: 2669 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col6 (type: string) - outputColumnNames: _col6 - Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col6 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Select Operator - expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - function name: json_tuple - Filter Operator - predicate: c0 is not null (type: boolean) - Statistics: Num rows: 6 Data size: 3185 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col6 (type: string) - outputColumnNames: _col6 - Statistics: Num rows: 12 Data size: 5854 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col6 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 -PREHOOK: type: QUERY -PREHOOK: Input: default@json_t -#### A masked pattern was here #### -POSTHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@json_t -#### A masked pattern was here #### -2 2 -value2 1 -NULL 1 -PREHOOK: query: CREATE TABLE dest1_n65(c1 STRING) STORED AS RCFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n65 -POSTHOOK: query: CREATE TABLE dest1_n65(c1 STRING) STORED AS RCFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n65 -PREHOOK: query: INSERT OVERWRITE TABLE dest1_n65 SELECT '{"a":"b\nc"}' FROM src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n65 -POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n65 SELECT '{"a":"b\nc"}' FROM src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n65 -POSTHOOK: Lineage: dest1_n65.c1 SIMPLE [] -PREHOOK: query: SELECT * FROM dest1_n65 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n65 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM dest1_n65 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n65 -#### A masked pattern was here #### -{"a":"b -c"} -PREHOOK: query: SELECT json FROM dest1_n65 a LATERAL VIEW json_tuple(c1, 'a') b AS json -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n65 -#### A masked pattern was here #### -POSTHOOK: query: SELECT json FROM dest1_n65 a LATERAL VIEW json_tuple(c1, 'a') b AS json -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n65 -#### A masked pattern was here #### -b -c diff --git ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out deleted file mode 100644 index 073b6d48d9..0000000000 --- ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out +++ /dev/null @@ -1,428 +0,0 @@ -PREHOOK: query: create table url_t (key string, fullurl string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@url_t -POSTHOOK: query: create table url_t (key string, fullurl string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@url_t -PREHOOK: query: insert overwrite table url_t -select * from ( - select '1', 'http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1' from src tablesample (1 rows) - union all - select '2', 'https://www.socs.uts.edu.au:80/MosaicDocs-old/url-primer.html?k1=tps#chapter1' from src tablesample (1 rows) - union all - select '3', 'ftp://sites.google.com/a/example.com/site/page' from src tablesample (1 rows) - union all - select '4', cast(null as string) from src tablesample (1 rows) - union all - select '5', 'htttp://' from src tablesample (1 rows) - union all - select '6', '[invalid url string]' from src tablesample (1 rows) -) s -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@url_t -POSTHOOK: query: insert overwrite table url_t -select * from ( - select '1', 'http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1' from src tablesample (1 rows) - union all - select '2', 'https://www.socs.uts.edu.au:80/MosaicDocs-old/url-primer.html?k1=tps#chapter1' from src tablesample (1 rows) - union all - select '3', 'ftp://sites.google.com/a/example.com/site/page' from src tablesample (1 rows) - union all - select '4', cast(null as string) from src tablesample (1 rows) - union all - select '5', 'htttp://' from src tablesample (1 rows) - union all - select '6', '[invalid url string]' from src tablesample (1 rows) -) s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@url_t -POSTHOOK: Lineage: url_t.fullurl EXPRESSION [] -POSTHOOK: Lineage: url_t.key EXPRESSION [] -PREHOOK: query: describe function parse_url_tuple -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: describe function parse_url_tuple -POSTHOOK: type: DESCFUNCTION -parse_url_tuple(url, partname1, partname2, ..., partnameN) - extracts N (N>=1) parts from a URL. -It takes a URL and one or multiple partnames, and returns a tuple. All the input parameters and output column types are string. -PREHOOK: query: describe function extended parse_url_tuple -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: describe function extended parse_url_tuple -POSTHOOK: type: DESCFUNCTION -parse_url_tuple(url, partname1, partname2, ..., partnameN) - extracts N (N>=1) parts from a URL. -It takes a URL and one or multiple partnames, and returns a tuple. All the input parameters and output column types are string. -Partname: HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, USERINFO, QUERY: -Note: Partnames are case-sensitive, and should not contain unnecessary white spaces. -Example: - > SELECT b.* FROM src LATERAL VIEW parse_url_tuple(fullurl, 'HOST', 'PATH', 'QUERY', 'QUERY:id') b as host, path, query, query_id LIMIT 1; - > SELECT parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from src a; -Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDTFParseUrlTuple -Function type:BUILTIN -PREHOOK: query: explain -select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 3159 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 8598 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) - Select Operator - expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE - function name: parse_url_tuple - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 8598 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: string), VALUE._col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: select a.key, b.* from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -1 facebook.com /path1/p.php k1=v1&k2=v2 Ref1 http /path1/p.php?k1=v1&k2=v2 facebook.com NULL v1 -2 www.socs.uts.edu.au /MosaicDocs-old/url-primer.html k1=tps chapter1 https /MosaicDocs-old/url-primer.html?k1=tps www.socs.uts.edu.au:80 NULL tps -3 sites.google.com /a/example.com/site/page NULL NULL ftp /a/example.com/site/page sites.google.com NULL NULL -4 NULL NULL NULL NULL NULL NULL NULL NULL NULL -5 NULL NULL NULL NULL NULL NULL NULL NULL NULL -6 NULL NULL NULL NULL NULL NULL NULL NULL NULL -PREHOOK: query: explain -select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 5556 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 5556 Basic stats: COMPLETE Column stats: COMPLETE - function name: parse_url_tuple - Select Operator - expressions: c0 (type: string), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string), c5 (type: string), c6 (type: string), c7 (type: string), c8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - null sort order: zzz - sort order: +++ - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: select parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') as (ho, pa, qu, re, pr, fi, au, us, qk1) from url_t a order by ho, pa, qu -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -facebook.com /path1/p.php k1=v1&k2=v2 Ref1 http /path1/p.php?k1=v1&k2=v2 facebook.com NULL v1 -sites.google.com /a/example.com/site/page NULL NULL ftp /a/example.com/site/page sites.google.com NULL NULL -www.socs.uts.edu.au /MosaicDocs-old/url-primer.html k1=tps chapter1 https /MosaicDocs-old/url-primer.html?k1=tps www.socs.uts.edu.au:80 NULL tps -NULL NULL NULL NULL NULL NULL NULL NULL NULL -NULL NULL NULL NULL NULL NULL NULL NULL NULL -NULL NULL NULL NULL NULL NULL NULL NULL NULL -PREHOOK: query: explain -select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 6 Data size: 3159 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 12 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col7 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - Select Operator - expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string), 'host' (type: string), 'query' (type: string), 'QUERY:nonExistCol' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 6 Data size: 7107 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 7107 Basic stats: COMPLETE Column stats: COMPLETE - function name: parse_url_tuple - Lateral View Join Operator - outputColumnNames: _col0, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16 - Statistics: Num rows: 12 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col5 (type: string), _col7 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: select a.key, b.ho, b.qu, b.qk1, b.err1, b.err2, b.err3 from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1', 'host', 'query', 'QUERY:nonExistCol') b as ho, pa, qu, re, pr, fi, au, us, qk1, err1, err2, err3 order by a.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -1 facebook.com k1=v1&k2=v2 v1 NULL NULL NULL -2 www.socs.uts.edu.au k1=tps tps NULL NULL NULL -3 sites.google.com NULL NULL NULL NULL NULL -4 NULL NULL NULL NULL NULL NULL -5 NULL NULL NULL NULL NULL NULL -6 NULL NULL NULL NULL NULL NULL -PREHOOK: query: explain -select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: explain -select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 6 Data size: 2649 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string) - outputColumnNames: _col5 - Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col5 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Select Operator - expressions: fullurl (type: string), 'HOST' (type: string), 'PATH' (type: string), 'QUERY' (type: string), 'REF' (type: string), 'PROTOCOL' (type: string), 'FILE' (type: string), 'AUTHORITY' (type: string), 'USERINFO' (type: string), 'QUERY:k1' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE - function name: parse_url_tuple - Filter Operator - predicate: c8 is not null (type: boolean) - Statistics: Num rows: 6 Data size: 5439 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string) - outputColumnNames: _col5 - Statistics: Num rows: 12 Data size: 8088 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col5 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho -PREHOOK: type: QUERY -PREHOOK: Input: default@url_t -#### A masked pattern was here #### -POSTHOOK: query: select ho, count(*) from url_t a lateral view parse_url_tuple(a.fullurl, 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'FILE', 'AUTHORITY', 'USERINFO', 'QUERY:k1') b as ho, pa, qu, re, pr, fi, au, us, qk1 where qk1 is not null group by ho -POSTHOOK: type: QUERY -POSTHOOK: Input: default@url_t -#### A masked pattern was here #### -facebook.com 1 -www.socs.uts.edu.au 1 diff --git ql/src/test/results/clientpositive/udtf_stack.q.out ql/src/test/results/clientpositive/udtf_stack.q.out deleted file mode 100644 index bfae719f59..0000000000 --- ql/src/test/results/clientpositive/udtf_stack.q.out +++ /dev/null @@ -1,223 +0,0 @@ -PREHOOK: query: DESCRIBE FUNCTION stack -PREHOOK: type: DESCFUNCTION -POSTHOOK: query: DESCRIBE FUNCTION stack -POSTHOOK: type: DESCFUNCTION -stack(n, cols...) - turns k columns into n rows of size k/n each -PREHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6 - Statistics: Num rows: 1000 Data size: 283000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string), _col6 (type: array) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Select Operator - expressions: 2 (type: int), 'x' (type: string), array(1) (type: array), 'z' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 111000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 111000 Basic stats: COMPLETE Column stats: COMPLETE - function name: stack - Lateral View Join Operator - outputColumnNames: _col5, _col6 - Statistics: Num rows: 1000 Data size: 283000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string), _col6 (type: array) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 2 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 172000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col5, _col6 - Statistics: Num rows: 1000 Data size: 307000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string), _col6 (type: array) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Select Operator - expressions: 2 (type: int), 'x' (type: string), array(1) (type: array), 'z' (type: string), array(4) (type: array) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 135000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 500 Data size: 135000 Basic stats: COMPLETE Column stats: COMPLETE - function name: stack - Lateral View Join Operator - outputColumnNames: _col5, _col6 - Statistics: Num rows: 1000 Data size: 307000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: string), _col6 (type: array) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 2 - Processor Tree: - ListSink - -PREHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -x [1] -z NULL -PREHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -x [1] -z [4] -PREHOOK: query: EXPLAIN -SELECT stack(1, "en", "dbpedia", NULL ) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT stack(1, "en", "dbpedia", NULL ) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int), 'en' (type: string), 'dbpedia' (type: string), null (type: void) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: COMPLETE - function name: stack - Select Operator - expressions: col0 (type: string), col1 (type: string), null (type: void) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -en dbpedia NULL diff --git ql/src/test/results/clientpositive/union10.q.out ql/src/test/results/clientpositive/union10.q.out deleted file mode 100644 index 9c31c86c07..0000000000 --- ql/src/test/results/clientpositive/union10.q.out +++ /dev/null @@ -1,348 +0,0 @@ -PREHOOK: query: create table tmptable_n3(key string, value int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tmptable_n3 -POSTHOOK: query: create table tmptable_n3(key string, value int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmptable_n3 -PREHOOK: query: explain -insert overwrite table tmptable_n3 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@tmptable_n3 -POSTHOOK: query: explain -insert overwrite table tmptable_n3 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@tmptable_n3 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-9 is a root stage - Stage-10 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: string, int - Table: default.tmptable_n3 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n3 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst3' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: insert overwrite table tmptable_n3 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@tmptable_n3 -POSTHOOK: query: insert overwrite table tmptable_n3 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@tmptable_n3 -POSTHOOK: Lineage: tmptable_n3.key EXPRESSION [] -POSTHOOK: Lineage: tmptable_n3.value EXPRESSION [(src)s1.null, (src)s2.null, (src)s3.null, ] -PREHOOK: query: select * from tmptable_n3 x sort by x.key -PREHOOK: type: QUERY -PREHOOK: Input: default@tmptable_n3 -#### A masked pattern was here #### -POSTHOOK: query: select * from tmptable_n3 x sort by x.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tmptable_n3 -#### A masked pattern was here #### -tst1 500 -tst2 500 -tst3 500 diff --git ql/src/test/results/clientpositive/union11.q.out ql/src/test/results/clientpositive/union11.q.out deleted file mode 100644 index 1e7db239d2..0000000000 --- ql/src/test/results/clientpositive/union11.q.out +++ /dev/null @@ -1,237 +0,0 @@ -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 - Stage-3 is a root stage - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst3' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src s2 - UNION ALL - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -tst1 1 -tst2 1 -tst3 1 diff --git ql/src/test/results/clientpositive/union12.q.out ql/src/test/results/clientpositive/union12.q.out deleted file mode 100644 index f2b48413c6..0000000000 --- ql/src/test/results/clientpositive/union12.q.out +++ /dev/null @@ -1,356 +0,0 @@ -PREHOOK: query: create table tmptable_n10(key string, value int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tmptable_n10 -POSTHOOK: query: create table tmptable_n10(key string, value int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmptable_n10 -PREHOOK: query: explain -insert overwrite table tmptable_n10 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src1 s2 - UNION ALL - select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -PREHOOK: Input: default@srcbucket -PREHOOK: Output: default@tmptable_n10 -POSTHOOK: query: explain -insert overwrite table tmptable_n10 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src1 s2 - UNION ALL - select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -POSTHOOK: Input: default@srcbucket -POSTHOOK: Output: default@tmptable_n10 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-9 is a root stage - Stage-10 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, value - Statistics: Num rows: 3 Data size: 276 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: string, int - Table: default.tmptable_n10 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n10 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst3' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: insert overwrite table tmptable_n10 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src1 s2 - UNION ALL - select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -PREHOOK: Input: default@srcbucket -PREHOOK: Output: default@tmptable_n10 -POSTHOOK: query: insert overwrite table tmptable_n10 - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION ALL - select 'tst2' as key, count(1) as value from src1 s2 - UNION ALL - select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -POSTHOOK: Input: default@srcbucket -POSTHOOK: Output: default@tmptable_n10 -POSTHOOK: Lineage: tmptable_n10.key EXPRESSION [] -POSTHOOK: Lineage: tmptable_n10.value EXPRESSION [(src)s1.null, (src1)s2.null, (srcbucket)s3.null, ] -PREHOOK: query: select * from tmptable_n10 x sort by x.key -PREHOOK: type: QUERY -PREHOOK: Input: default@tmptable_n10 -#### A masked pattern was here #### -POSTHOOK: query: select * from tmptable_n10 x sort by x.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tmptable_n10 -#### A masked pattern was here #### -tst1 500 -tst2 25 -tst3 1000 diff --git ql/src/test/results/clientpositive/union14.q.out ql/src/test/results/clientpositive/union14.q.out deleted file mode 100644 index 39c7ff06b2..0000000000 --- ql/src/test/results/clientpositive/union14.q.out +++ /dev/null @@ -1,160 +0,0 @@ -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 26 Data size: 2238 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 26 Data size: 2238 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### - 10 -128 1 -146 1 -150 1 -213 1 -224 1 -238 1 -255 1 -273 1 -278 1 -311 1 -369 1 -401 1 -406 1 -66 1 -98 1 -tst1 1 diff --git ql/src/test/results/clientpositive/union15.q.out ql/src/test/results/clientpositive/union15.q.out deleted file mode 100644 index 947d81b21f..0000000000 --- ql/src/test/results/clientpositive/union15.q.out +++ /dev/null @@ -1,187 +0,0 @@ -PREHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 51 Data size: 4388 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 51 Data size: 4388 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: s3 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 51 Data size: 4388 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src1 s2 - UNION ALL - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src1 -#### A masked pattern was here #### - 20 -128 2 -146 2 -150 2 -213 2 -224 2 -238 2 -255 2 -273 2 -278 2 -311 2 -369 2 -401 2 -406 2 -66 2 -98 2 -tst1 1 diff --git ql/src/test/results/clientpositive/union16.q.out ql/src/test/results/clientpositive/union16.q.out deleted file mode 100644 index 2c8c33e500..0000000000 --- ql/src/test/results/clientpositive/union16.q.out +++ /dev/null @@ -1,663 +0,0 @@ -PREHOOK: query: EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src) src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src) src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 12500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT count(1) FROM ( - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src) src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT count(1) FROM ( - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src UNION ALL - SELECT key, value FROM src) src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -12500 diff --git ql/src/test/results/clientpositive/union20.q.out ql/src/test/results/clientpositive/union20.q.out deleted file mode 100644 index 9d91b76993..0000000000 --- ql/src/test/results/clientpositive/union20.q.out +++ /dev/null @@ -1,240 +0,0 @@ -PREHOOK: query: explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - TableScan - alias: s2 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - TableScan - Union - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - TableScan - alias: s4 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 167 Data size: 29820 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 168 Data size: 91392 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 168 Data size: 91392 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), CAST( _col0 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -0 val_0 0 val_0 -2 val_2 2 val_2 -4 val_4 4 val_4 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -5 val_5 5 val_5 -8 val_8 8 val_8 -9 val_9 9 val_9 -tst1 500 tst1 500 diff --git ql/src/test/results/clientpositive/union21.q.out ql/src/test/results/clientpositive/union21.q.out deleted file mode 100644 index ee8c97197b..0000000000 --- ql/src/test/results/clientpositive/union21.q.out +++ /dev/null @@ -1,750 +0,0 @@ -PREHOOK: query: explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION ALL - SELECT reverse(key) as key from src - UNION ALL - SELECT key as key from src - UNION ALL - SELECT astring as key from src_thrift - UNION ALL - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src_thrift -#### A masked pattern was here #### -POSTHOOK: query: explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION ALL - SELECT reverse(key) as key from src - UNION ALL - SELECT key as key from src - UNION ALL - SELECT astring as key from src_thrift - UNION ALL - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src_thrift -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '1' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1522 Data size: 201144 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: reverse(key) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1522 Data size: 201144 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: bigint) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1522 Data size: 201144 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: bigint) - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: astring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1522 Data size: 201144 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: bigint) - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: lstring[0] (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1522 Data size: 201144 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 316 Data size: 60672 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION ALL - SELECT reverse(key) as key from src - UNION ALL - SELECT key as key from src - UNION ALL - SELECT astring as key from src_thrift - UNION ALL - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@src_thrift -#### A masked pattern was here #### -POSTHOOK: query: SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION ALL - SELECT reverse(key) as key from src - UNION ALL - SELECT key as key from src - UNION ALL - SELECT astring as key from src_thrift - UNION ALL - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@src_thrift -#### A masked pattern was here #### -0 7 -001 2 -002 2 -004 1 -01 1 -013 1 -02 1 -021 2 -03 1 -032 5 -034 3 -051 1 -061 1 -062 1 -063 1 -064 1 -07 3 -071 1 -074 1 -08 1 -081 1 -082 2 -084 3 -09 3 -091 1 -094 1 -1 500 -10 2 -100 2 -102 1 -103 2 -104 7 -105 1 -11 2 -111 2 -113 5 -114 2 -116 1 -118 2 -119 3 -12 2 -120 2 -122 2 -123 2 -124 1 -125 2 -126 1 -128 3 -129 2 -131 2 -133 3 -134 5 -136 1 -137 2 -138 4 -14 1 -142 1 -143 2 -145 1 -146 2 -149 2 -15 4 -150 1 -152 2 -153 2 -155 1 -156 1 -157 1 -158 1 -160 1 -162 1 -163 1 -164 2 -165 2 -166 1 -167 3 -168 1 -169 4 -17 1 -170 1 -172 2 -174 2 -175 2 -176 2 -177 1 -178 1 -179 2 -18 2 -180 1 -181 2 -182 2 -183 1 -184 1 -186 1 -187 3 -189 1 -19 1 -190 1 -191 4 -192 2 -193 3 -194 2 -195 2 -196 1 -197 2 -199 3 -2 2 -20 2 -200 2 -201 1 -202 2 -203 3 -204 1 -205 2 -207 2 -208 3 -209 2 -21 2 -213 2 -214 1 -216 2 -217 2 -218 1 -219 2 -221 2 -222 2 -223 4 -224 2 -226 1 -228 1 -229 2 -230 5 -233 3 -234 1 -235 1 -237 2 -238 2 -239 2 -24 4 -241 1 -242 4 -243 2 -244 1 -247 1 -248 1 -249 1 -251 2 -252 2 -254 1 -255 2 -256 2 -257 1 -258 1 -26 2 -260 1 -261 1 -262 2 -263 2 -264 2 -265 2 -266 1 -27 3 -271 2 -272 4 -273 3 -274 2 -275 1 -277 4 -278 2 -28 2 -280 2 -281 2 -282 4 -283 3 -284 2 -285 1 -286 1 -287 1 -288 2 -289 1 -29 1 -291 2 -292 2 -293 1 -294 2 -296 1 -298 3 -30 2 -301 2 -302 3 -304 3 -305 1 -306 1 -307 2 -308 1 -309 2 -310 1 -311 5 -312 2 -314 2 -315 1 -316 3 -317 2 -318 3 -321 2 -322 4 -323 2 -325 2 -327 3 -33 2 -331 3 -332 3 -333 4 -335 1 -336 1 -338 1 -339 1 -34 2 -341 2 -342 2 -344 3 -345 1 -348 5 -35 4 -351 2 -353 4 -354 1 -356 1 -360 1 -361 1 -362 2 -364 3 -365 1 -366 1 -367 2 -368 1 -369 3 -37 2 -372 3 -373 2 -374 1 -375 1 -377 1 -378 1 -379 1 -38 2 -381 1 -382 3 -384 4 -386 1 -389 1 -391 3 -392 1 -393 2 -394 2 -395 2 -396 3 -397 2 -399 2 -4 2 -40 1 -400 1 -401 7 -402 1 -403 3 -404 4 -406 4 -407 1 -409 3 -41 1 -411 2 -412 1 -413 2 -414 4 -417 3 -418 1 -419 1 -42 4 -421 1 -422 2 -424 4 -427 1 -429 2 -43 2 -430 3 -431 5 -432 1 -435 1 -436 1 -437 1 -438 3 -439 2 -44 2 -442 1 -443 3 -444 2 -446 1 -448 1 -449 1 -45 1 -452 1 -453 1 -454 6 -455 1 -457 1 -458 2 -459 2 -46 1 -460 1 -461 2 -462 2 -463 3 -466 3 -467 1 -468 4 -469 5 -47 2 -470 1 -471 2 -472 2 -473 1 -475 1 -477 1 -478 2 -479 1 -48 2 -480 3 -481 1 -482 2 -483 4 -484 2 -485 1 -487 1 -489 4 -490 1 -491 2 -492 2 -493 2 -494 2 -495 1 -496 1 -497 1 -498 3 -5 6 -50 1 -501 1 -502 2 -503 1 -51 4 -513 1 -521 2 -523 2 -53 4 -532 1 -533 1 -534 1 -54 1 -541 1 -543 1 -551 1 -552 2 -554 1 -56 1 -561 2 -562 2 -563 1 -57 1 -571 2 -572 1 -573 1 -574 1 -58 3 -582 1 -584 1 -59 2 -591 2 -593 2 -594 1 -60 1 -603 1 -604 4 -611 1 -612 2 -613 3 -62 2 -621 1 -622 1 -631 1 -633 1 -634 1 -64 1 -641 2 -644 1 -65 1 -651 1 -652 2 -653 1 -66 2 -661 1 -662 1 -663 1 -664 3 -67 4 -671 2 -68 1 -681 1 -682 1 -683 1 -69 2 -691 1 -692 1 -693 3 -694 1 -70 4 -702 2 -703 2 -704 1 -71 1 -712 2 -713 2 -714 3 -72 3 -723 3 -724 1 -73 2 -731 2 -732 2 -734 1 -74 2 -742 1 -75 1 -751 1 -752 1 -754 1 -76 4 -761 3 -763 2 -764 1 -77 2 -771 1 -772 4 -773 1 -774 1 -78 2 -781 3 -782 1 -784 1 -79 2 -791 2 -793 2 -794 1 -8 2 -80 2 -802 3 -803 1 -81 2 -811 2 -812 1 -813 3 -814 1 -82 2 -821 3 -822 1 -83 2 -831 4 -832 2 -833 1 -834 3 -84 2 -842 1 -843 5 -844 1 -85 3 -851 1 -852 1 -854 2 -86 1 -861 1 -863 1 -864 4 -87 2 -871 1 -872 2 -873 1 -874 2 -882 2 -89 2 -892 3 -894 3 -9 2 -90 4 -902 2 -903 2 -904 3 -91 1 -911 3 -912 2 -914 1 -92 1 -921 2 -922 2 -924 2 -932 2 -933 1 -934 2 -941 2 -942 1 -944 1 -95 2 -954 2 -96 2 -961 4 -963 3 -964 5 -97 2 -971 2 -973 1 -974 1 -98 2 -981 1 -982 1 -983 1 -984 4 -991 3 -993 2 -NULL 2 -record_0 1 -record_1 1 -record_2 1 -record_3 1 -record_4 1 -record_5 1 -record_6 1 -record_7 1 -record_8 1 -record_9 1 diff --git ql/src/test/results/clientpositive/union24.q.out ql/src/test/results/clientpositive/union24.q.out deleted file mode 100644 index 32a86e7f02..0000000000 --- ql/src/test/results/clientpositive/union24.q.out +++ /dev/null @@ -1,1713 +0,0 @@ -PREHOOK: query: create table src2_n6 as select key, count(1) as count from src group by key -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@src2_n6 -POSTHOOK: query: create table src2_n6 as select key, count(1) as count from src group by key -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src2_n6 -POSTHOOK: Lineage: src2_n6.count EXPRESSION [(src)src.null, ] -POSTHOOK: Lineage: src2_n6.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: create table src3_n2 as select * from src2_n6 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2_n6 -PREHOOK: Output: database:default -PREHOOK: Output: default@src3_n2 -POSTHOOK: query: create table src3_n2 as select * from src2_n6 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src3_n2 -POSTHOOK: Lineage: src3_n2.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src3_n2.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: create table src4_n0 as select * from src2_n6 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2_n6 -PREHOOK: Output: database:default -PREHOOK: Output: default@src4_n0 -POSTHOOK: query: create table src4_n0 as select * from src2_n6 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src4_n0 -POSTHOOK: Lineage: src4_n0.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src4_n0.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: create table src5_n3 as select * from src2_n6 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src2_n6 -PREHOOK: Output: database:default -PREHOOK: Output: default@src5_n3 -POSTHOOK: query: create table src5_n3 as select * from src2_n6 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src5_n3 -POSTHOOK: Lineage: src5_n3.count SIMPLE [(src2_n6)src2_n6.FieldSchema(name:count, type:bigint, comment:null), ] -POSTHOOK: Lineage: src5_n3.key SIMPLE [(src2_n6)src2_n6.FieldSchema(name:key, type:string, comment:null), ] -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select key, count from src4_n0 where key < 10 - union all - select key, count(1) as count from src5_n3 where key < 10 group by key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select key, count from src4_n0 where key < 10 - union all - select key, count(1) as count from src5_n3 where key < 10 group by key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -OPTIMIZED SQL: SELECT `key`, `count` -FROM `default`.`src2_n6` -WHERE `key` < 10 -UNION ALL -SELECT `key`, `count` -FROM `default`.`src3_n2` -WHERE `key` < 10 -UNION ALL -SELECT `key`, `count` -FROM `default`.`src4_n0` -WHERE `key` < 10 -UNION ALL -SELECT `key`, COUNT(*) AS `$f1` -FROM `default`.`src5_n3` -WHERE `key` < 10 -GROUP BY `key` -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - alias: src5_n3 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src5_n3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5_n3 - name: default.src5_n3 - Truncated Path -> Alias: - /src5_n3 [$hdt$_2-subquery2:src5_n3] - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src2_n6 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - alias: src3_n2 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - alias: src4_n0 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - GatherStats: false - Union - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 360 Data size: 34200 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10004 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -#### A masked pattern was here #### - Partition - base file name: src2_n6 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2_n6 - name: default.src2_n6 -#### A masked pattern was here #### - Partition - base file name: src3_n2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3_n2 - name: default.src3_n2 -#### A masked pattern was here #### - Partition - base file name: src4_n0 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4_n0 - name: default.src4_n0 - Truncated Path -> Alias: - /src2_n6 [$hdt$_2-subquery1-subquery1-subquery1:src2_n6] - /src3_n2 [$hdt$_2-subquery1-subquery1-subquery2:src3_n2] - /src4_n0 [$hdt$_2-subquery1-subquery2:src4_n0] -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select key, count from src4_n0 where key < 10 - union all - select key, count(1) as count from src5_n3 where key < 10 group by key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select key, count from src4_n0 where key < 10 - union all - select key, count(1) as count from src5_n3 where key < 10 group by key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -0 1 -0 3 -0 3 -0 3 -2 1 -2 1 -2 1 -2 1 -4 1 -4 1 -4 1 -4 1 -5 1 -5 3 -5 3 -5 3 -8 1 -8 1 -8 1 -8 1 -9 1 -9 1 -9 1 -9 1 -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -OPTIMIZED SQL: SELECT `key`, `count` -FROM `default`.`src2_n6` -WHERE `key` < 10 -UNION ALL -SELECT `key`, `count` -FROM `default`.`src3_n2` -WHERE `key` < 10 -UNION ALL -SELECT `t4`.`key`, `t6`.`count` -FROM (SELECT `key` -FROM `default`.`src4_n0` -WHERE `key` < 10) AS `t4` -INNER JOIN (SELECT `key`, `count` -FROM `default`.`src5_n3` -WHERE `key` < 10) AS `t6` ON `t4`.`key` = `t6`.`key` -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-2 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - auto parallelism: false - TableScan - alias: b - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: _col1 (type: bigint) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4_n0 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4_n0 - name: default.src4_n0 -#### A masked pattern was here #### - Partition - base file name: src5_n3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5_n3 - name: default.src5_n3 - Truncated Path -> Alias: - /src4_n0 [$hdt$_1-subquery2:$hdt$_1:a] - /src5_n3 [$hdt$_1-subquery2:$hdt$_2:b] - Needs Tagging: true - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src2_n6 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - alias: src3_n2 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - GatherStats: false - Union - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10004 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -#### A masked pattern was here #### - Partition - base file name: src2_n6 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2_n6 - name: default.src2_n6 -#### A masked pattern was here #### - Partition - base file name: src3_n2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3_n2 - name: default.src3_n2 - Truncated Path -> Alias: - /src2_n6 [$hdt$_1-subquery1-subquery1:src2_n6] - /src3_n2 [$hdt$_1-subquery1-subquery2:src3_n2] -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, b.count as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -0 3 -0 3 -0 3 -2 1 -2 1 -2 1 -4 1 -4 1 -4 1 -5 3 -5 3 -5 3 -8 1 -8 1 -8 1 -9 1 -9 1 -9 1 -PREHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: explain extended -select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -OPTIMIZED SQL: SELECT `key`, `count` -FROM `default`.`src2_n6` -WHERE `key` < 10 -UNION ALL -SELECT `key`, `count` -FROM `default`.`src3_n2` -WHERE `key` < 10 -UNION ALL -SELECT `t4`.`key`, COUNT(*) AS `$f1` -FROM (SELECT `key` -FROM `default`.`src4_n0` -WHERE `key` < 10) AS `t4` -INNER JOIN (SELECT `key` -FROM `default`.`src5_n3` -WHERE `key` < 10) AS `t6` ON `t4`.`key` = `t6`.`key` -GROUP BY `t4`.`key` -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-5 depends on stages: Stage-4 - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - auto parallelism: false - TableScan - alias: b - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 26883 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src4_n0 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src4_n0 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src4_n0 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src4_n0 - name: default.src4_n0 -#### A masked pattern was here #### - Partition - base file name: src5_n3 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src5_n3 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src5_n3 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src5_n3 - name: default.src5_n3 - Truncated Path -> Alias: - /src4_n0 [$hdt$_1-subquery2:$hdt$_1:a] - /src5_n3 [$hdt$_1-subquery2:$hdt$_2:b] - Needs Tagging: true - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 103 Data size: 8961 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: bigint) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10004 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 51 Data size: 4845 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 1 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src2_n6 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - alias: src3_n2 - filterExpr: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 10.0D) (type: boolean) - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), count (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - TableScan - GatherStats: false - Union - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 257 Data size: 24415 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10005 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,bigint - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -#### A masked pattern was here #### - Partition - base file name: src2_n6 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src2_n6 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src2_n6 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src2_n6 - name: default.src2_n6 -#### A masked pattern was here #### - Partition - base file name: src3_n2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"count":"true","key":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,count - columns.comments - columns.types string:bigint -#### A masked pattern was here #### - name default.src3_n2 - numFiles 1 - numRows 309 - rawDataSize 1482 - serialization.ddl struct src3_n2 { string key, i64 count} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1791 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src3_n2 - name: default.src3_n2 - Truncated Path -> Alias: - /src2_n6 [$hdt$_1-subquery1-subquery1:src2_n6] - /src3_n2 [$hdt$_1-subquery1-subquery2:src3_n2] -#### A masked pattern was here #### - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key -)s -PREHOOK: type: QUERY -PREHOOK: Input: default@src2_n6 -PREHOOK: Input: default@src3_n2 -PREHOOK: Input: default@src4_n0 -PREHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -POSTHOOK: query: select s.key, s.count from ( - select key, count from src2_n6 where key < 10 - union all - select key, count from src3_n2 where key < 10 - union all - select a.key as key, count(1) as count from src4_n0 a join src5_n3 b on a.key=b.key where a.key < 10 group by a.key -)s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src2_n6 -POSTHOOK: Input: default@src3_n2 -POSTHOOK: Input: default@src4_n0 -POSTHOOK: Input: default@src5_n3 -#### A masked pattern was here #### -0 1 -0 3 -0 3 -2 1 -2 1 -2 1 -4 1 -4 1 -4 1 -5 1 -5 3 -5 3 -8 1 -8 1 -8 1 -9 1 -9 1 -9 1 diff --git ql/src/test/results/clientpositive/union25.q.out ql/src/test/results/clientpositive/union25.q.out deleted file mode 100644 index 82b0b01bb0..0000000000 --- ql/src/test/results/clientpositive/union25.q.out +++ /dev/null @@ -1,269 +0,0 @@ -PREHOOK: query: create table tmp_srcpart like srcpart -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tmp_srcpart -POSTHOOK: query: create table tmp_srcpart like srcpart -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmp_srcpart -PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') -select key, value from srcpart where ds='2008-04-08' and hr='11' -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 -POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') -select key, value from srcpart where ds='2008-04-08' and hr='11' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain -create table tmp_unionall as -SELECT count(1) as counts, key, value -FROM -( - SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - - UNION ALL - - SELECT key, key as value FROM ( - SELECT distinct key FROM ( - SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - UNION ALL - SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' - )t - ) master_table -) a GROUP BY key, value -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@tmp_srcpart -PREHOOK: Input: default@tmp_srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: database:default -PREHOOK: Output: default@tmp_unionall -POSTHOOK: query: explain -create table tmp_unionall as -SELECT count(1) as counts, key, value -FROM -( - SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - - UNION ALL - - SELECT key, key as value FROM ( - SELECT distinct key FROM ( - SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' - UNION ALL - SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' - )t - ) master_table -) a GROUP BY key, value -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@tmp_srcpart -POSTHOOK: Input: default@tmp_srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tmp_unionall -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - Stage-7 depends on stages: Stage-0, Stage-4 - Stage-3 depends on stages: Stage-7 - Stage-4 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: b - filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 316 Data size: 27492 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 316 Data size: 54984 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 816 Data size: 143984 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - TableScan - Union - Statistics: Num rows: 816 Data size: 143984 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmp_unionall - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string) - outputColumnNames: col1, col2, col3 - Statistics: Num rows: 408 Data size: 75888 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll'), compute_stats(col3, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-7 - Create Table - columns: counts bigint, key string, value string - name: default.tmp_unionall - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: counts, key, value - Column Types: bigint, string, string - Table: default.tmp_unionall - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1304 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - diff --git ql/src/test/results/clientpositive/union27.q.out ql/src/test/results/clientpositive/union27.q.out deleted file mode 100644 index 295ddf3787..0000000000 --- ql/src/test/results/clientpositive/union27.q.out +++ /dev/null @@ -1,160 +0,0 @@ -PREHOOK: query: create table jackson_sev_same_n0 as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@jackson_sev_same_n0 -POSTHOOK: query: create table jackson_sev_same_n0 as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@jackson_sev_same_n0 -POSTHOOK: Lineage: jackson_sev_same_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: jackson_sev_same_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table dim_pho_n0 as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@dim_pho_n0 -POSTHOOK: query: create table dim_pho_n0 as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dim_pho_n0 -POSTHOOK: Lineage: dim_pho_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dim_pho_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table jackson_sev_add_n0 as select * from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@jackson_sev_add_n0 -POSTHOOK: query: create table jackson_sev_add_n0 as select * from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@jackson_sev_add_n0 -POSTHOOK: Lineage: jackson_sev_add_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: jackson_sev_add_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 -PREHOOK: type: QUERY -PREHOOK: Input: default@dim_pho_n0 -PREHOOK: Input: default@jackson_sev_add_n0 -PREHOOK: Input: default@jackson_sev_same_n0 -#### A masked pattern was here #### -POSTHOOK: query: explain select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dim_pho_n0 -POSTHOOK: Input: default@jackson_sev_add_n0 -POSTHOOK: Input: default@jackson_sev_same_n0 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: dim_pho_n0 - filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - TableScan - alias: jackson_sev_add_n0 - filterExpr: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 97.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 -PREHOOK: type: QUERY -PREHOOK: Input: default@dim_pho_n0 -PREHOOK: Input: default@jackson_sev_add_n0 -PREHOOK: Input: default@jackson_sev_same_n0 -#### A masked pattern was here #### -POSTHOOK: query: select b.* from jackson_sev_same_n0 a join (select * from dim_pho_n0 union all select * from jackson_sev_add_n0)b on a.key=b.key and b.key=97 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dim_pho_n0 -POSTHOOK: Input: default@jackson_sev_add_n0 -POSTHOOK: Input: default@jackson_sev_same_n0 -#### A masked pattern was here #### -97 val_97 -97 val_97 -97 val_97 -97 val_97 -97 val_97 -97 val_97 -97 val_97 -97 val_97 diff --git ql/src/test/results/clientpositive/union28.q.out ql/src/test/results/clientpositive/union28.q.out deleted file mode 100644 index f32db16960..0000000000 --- ql/src/test/results/clientpositive/union28.q.out +++ /dev/null @@ -1,351 +0,0 @@ -PREHOOK: query: create table union_subq_union(key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union -POSTHOOK: query: create table union_subq_union(key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union -PREHOOK: query: explain -insert overwrite table union_subq_union -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union -POSTHOOK: query: explain -insert overwrite table union_subq_union -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union -STAGE DEPENDENCIES: - Stage-9 is a root stage - Stage-2 depends on stages: Stage-9, Stage-10 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-10 is a root stage - -STAGE PLANS: - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.union_subq_union - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: insert overwrite table union_subq_union -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union -POSTHOOK: query: insert overwrite table union_subq_union -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union -POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union order by key, value limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union -#### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union order by key, value limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -2 val_2 -2 val_2 -2 val_2 -4 val_4 -4 val_4 -4 val_4 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -8 val_8 -8 val_8 -8 val_8 -9 val_9 diff --git ql/src/test/results/clientpositive/union29.q.out ql/src/test/results/clientpositive/union29.q.out deleted file mode 100644 index c8b76d5b77..0000000000 --- ql/src/test/results/clientpositive/union29.q.out +++ /dev/null @@ -1,289 +0,0 @@ -PREHOOK: query: create table union_subq_union_n1(key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union_n1 -POSTHOOK: query: create table union_subq_union_n1(key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union_n1 -PREHOOK: query: explain -insert overwrite table union_subq_union_n1 -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value from src - union all - select key, value from src - ) subq -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union_n1 -POSTHOOK: query: explain -insert overwrite table union_subq_union_n1 -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value from src - union all - select key, value from src - ) subq -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.union_subq_union_n1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n1 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: insert overwrite table union_subq_union_n1 -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value from src - union all - select key, value from src - ) subq -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union_n1 -POSTHOOK: query: insert overwrite table union_subq_union_n1 -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value from src - union all - select key, value from src - ) subq -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union_n1 -POSTHOOK: Lineage: union_subq_union_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union_n1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union_n1 order by key, value limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union_n1 -#### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union_n1 order by key, value limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union_n1 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -2 val_2 -2 val_2 -2 val_2 -4 val_4 -4 val_4 -4 val_4 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 diff --git ql/src/test/results/clientpositive/union30.q.out ql/src/test/results/clientpositive/union30.q.out deleted file mode 100644 index dc356b7b2c..0000000000 --- ql/src/test/results/clientpositive/union30.q.out +++ /dev/null @@ -1,415 +0,0 @@ -PREHOOK: query: create table union_subq_union_n2(key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union_subq_union_n2 -POSTHOOK: query: create table union_subq_union_n2(key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union_subq_union_n2 -PREHOOK: query: explain -insert overwrite table union_subq_union_n2 -select * from ( - -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a - -union all - -select key, value from src -) aa -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union_n2 -POSTHOOK: query: explain -insert overwrite table union_subq_union_n2 -select * from ( - -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a - -union all - -select key, value from src -) aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union_n2 -STAGE DEPENDENCIES: - Stage-10 is a root stage - Stage-2 depends on stages: Stage-10, Stage-11 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-11 is a root stage - -STAGE PLANS: - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 1500 Data size: 267000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 1500 Data size: 142500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.union_subq_union_n2 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union_n2 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-11 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: insert overwrite table union_subq_union_n2 -select * from ( - -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a - -union all - -select key, value from src -) aa -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@union_subq_union_n2 -POSTHOOK: query: insert overwrite table union_subq_union_n2 -select * from ( - -select * from ( - select key, value from src - union all - select key, value from - ( - select key, value, count(1) from src group by key, value - union all - select key, value, count(1) from src group by key, value - ) subq -) a - -union all - -select key, value from src -) aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@union_subq_union_n2 -POSTHOOK: Lineage: union_subq_union_n2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: union_subq_union_n2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from union_subq_union_n2 order by key, value limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@union_subq_union_n2 -#### A masked pattern was here #### -POSTHOOK: query: select * from union_subq_union_n2 order by key, value limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@union_subq_union_n2 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -2 val_2 -2 val_2 -2 val_2 -2 val_2 -4 val_4 -4 val_4 -4 val_4 -4 val_4 -5 val_5 -5 val_5 -5 val_5 -5 val_5 diff --git ql/src/test/results/clientpositive/union31.q.out ql/src/test/results/clientpositive/union31.q.out deleted file mode 100644 index 04760f69f1..0000000000 --- ql/src/test/results/clientpositive/union31.q.out +++ /dev/null @@ -1,1182 +0,0 @@ -PREHOOK: query: drop table t1_n28 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table t1_n28 -POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table t2_n19 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table t2_n19 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table t1_n28 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_n28 -POSTHOOK: query: create table t1_n28 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_n28 -POSTHOOK: Lineage: t1_n28.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1_n28.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t2_n19 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2_n19 -POSTHOOK: query: create table t2_n19 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2_n19 -POSTHOOK: Lineage: t2_n19.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t2_n19.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t3_n6(key string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t3_n6 -POSTHOOK: query: create table t3_n6(key string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t3_n6 -PREHOOK: query: create table t4_n2(value string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t4_n2 -POSTHOOK: query: create table t4_n2(value string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t4_n2 -PREHOOK: query: explain -from -(select * from t1_n28 - union all - select * from t2_n19 -) x -insert overwrite table t3_n6 - select key, count(1) group by key -insert overwrite table t4_n2 - select value, count(1) group by value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t3_n6 -PREHOOK: Output: default@t4_n2 -POSTHOOK: query: explain -from -(select * from t1_n28 - union all - select * from t2_n19 -) x -insert overwrite table t3_n6 - select key, count(1) group by key -insert overwrite table t4_n2 - select value, count(1) group by value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t3_n6 -POSTHOOK: Output: default@t4_n2 -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4, Stage-7 - Stage-4 depends on stages: Stage-2 - Stage-6 depends on stages: Stage-1, Stage-4, Stage-7 - Stage-5 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-5 - Stage-7 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t1_n28 - Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - alias: t2_n19 - Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 20 Data size: 3480 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t3_n6 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: key, cnt - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t3_n6 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, cnt - Column Types: string, int - Table: default.t3_n6 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: value, cnt - Column Types: string, int - Table: default.t4_n2 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t4_n2 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: value, cnt - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(value, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t4_n2 - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: from -(select * from t1_n28 - union all - select * from t2_n19 -) x -insert overwrite table t3_n6 - select key, count(1) group by key -insert overwrite table t4_n2 - select value, count(1) group by value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t3_n6 -PREHOOK: Output: default@t4_n2 -POSTHOOK: query: from -(select * from t1_n28 - union all - select * from t2_n19 -) x -insert overwrite table t3_n6 - select key, count(1) group by key -insert overwrite table t4_n2 - select value, count(1) group by value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t3_n6 -POSTHOOK: Output: default@t4_n2 -POSTHOOK: Lineage: t3_n6.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -POSTHOOK: Lineage: t3_n6.key EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t4_n2.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -POSTHOOK: Lineage: t4_n2.value EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:value, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: select * from t3_n6 -PREHOOK: type: QUERY -PREHOOK: Input: default@t3_n6 -#### A masked pattern was here #### -POSTHOOK: query: select * from t3_n6 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t3_n6 -#### A masked pattern was here #### -0 6 -2 2 -4 2 -5 6 -8 2 -9 2 -PREHOOK: query: select * from t4_n2 -PREHOOK: type: QUERY -PREHOOK: Input: default@t4_n2 -#### A masked pattern was here #### -POSTHOOK: query: select * from t4_n2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t4_n2 -#### A masked pattern was here #### -val_0 6 -val_2 2 -val_4 2 -val_5 6 -val_8 2 -val_9 2 -PREHOOK: query: create table t5_n0(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t5_n0 -POSTHOOK: query: create table t5_n0(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t5_n0 -PREHOOK: query: create table t6_n0(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t6_n0 -POSTHOOK: query: create table t6_n0(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t6_n0 -PREHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, count(1) as cnt from t2_n19 group by key -) x -insert overwrite table t5_n0 - select c1, sum(cnt) group by c1 -insert overwrite table t6_n0 - select c1, sum(cnt) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t5_n0 -PREHOOK: Output: default@t6_n0 -POSTHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, count(1) as cnt from t2_n19 group by key -) x -insert overwrite table t5_n0 - select c1, sum(cnt) group by c1 -insert overwrite table t6_n0 - select c1, sum(cnt) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t5_n0 -POSTHOOK: Output: default@t6_n0 -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-8 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0, Stage-5, Stage-7 - Stage-5 depends on stages: Stage-3 - Stage-6 depends on stages: Stage-1, Stage-5, Stage-7 - Stage-1 depends on stages: Stage-3 - Stage-7 depends on stages: Stage-3 - Stage-8 is a root stage - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t1_n28 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Forward - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t5_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: c1, cnt - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t6_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: c1, cnt - Statistics: Num rows: 5 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t5_n0 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, cnt - Column Types: string, int - Table: default.t5_n0 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, cnt - Column Types: string, int - Table: default.t6_n0 - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t6_n0 - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Map Reduce - Map Operator Tree: - TableScan - alias: t2_n19 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, count(1) as cnt from t2_n19 group by key -) x -insert overwrite table t5_n0 - select c1, sum(cnt) group by c1 -insert overwrite table t6_n0 - select c1, sum(cnt) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t5_n0 -PREHOOK: Output: default@t6_n0 -POSTHOOK: query: from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, count(1) as cnt from t2_n19 group by key -) x -insert overwrite table t5_n0 - select c1, sum(cnt) group by c1 -insert overwrite table t6_n0 - select c1, sum(cnt) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t5_n0 -POSTHOOK: Output: default@t6_n0 -POSTHOOK: Lineage: t5_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t5_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -POSTHOOK: Lineage: t6_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t6_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -PREHOOK: query: select * from t5_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@t5_n0 -#### A masked pattern was here #### -POSTHOOK: query: select * from t5_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t5_n0 -#### A masked pattern was here #### -0 6 -2 2 -4 2 -5 6 -8 2 -9 2 -PREHOOK: query: select * from t6_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@t6_n0 -#### A masked pattern was here #### -POSTHOOK: query: select * from t6_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t6_n0 -#### A masked pattern was here #### -0 6 -2 2 -4 2 -5 6 -8 2 -9 2 -PREHOOK: query: drop table t1_n28 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1_n28 -PREHOOK: Output: default@t1_n28 -POSTHOOK: query: drop table t1_n28 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Output: default@t1_n28 -PREHOOK: query: drop table t2_n19 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t2_n19 -POSTHOOK: query: drop table t2_n19 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t2_n19 -PREHOOK: query: create table t1_n28 as select * from src where key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_n28 -POSTHOOK: query: create table t1_n28 as select * from src where key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_n28 -POSTHOOK: Lineage: t1_n28.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1_n28.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: create table t2_n19 as select key, count(1) as cnt from src where key < 10 group by key -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2_n19 -POSTHOOK: query: create table t2_n19 as select key, count(1) as cnt from src where key < 10 group by key -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2_n19 -POSTHOOK: Lineage: t2_n19.cnt EXPRESSION [(src)src.null, ] -POSTHOOK: Lineage: t2_n19.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: create table t7_n1(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t7_n1 -POSTHOOK: query: create table t7_n1(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t7_n1 -PREHOOK: query: create table t8_n0(c1 string, cnt int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t8_n0 -POSTHOOK: query: create table t8_n0(c1 string, cnt int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t8_n0 -PREHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, cnt from t2_n19 -) x -insert overwrite table t7_n1 - select c1, count(1) group by c1 -insert overwrite table t8_n0 - select c1, count(1) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t7_n1 -PREHOOK: Output: default@t8_n0 -POSTHOOK: query: explain -from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, cnt from t2_n19 -) x -insert overwrite table t7_n1 - select c1, count(1) group by c1 -insert overwrite table t8_n0 - select c1, count(1) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t7_n1 -POSTHOOK: Output: default@t8_n0 -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0, Stage-5, Stage-7 - Stage-5 depends on stages: Stage-3 - Stage-6 depends on stages: Stage-1, Stage-5, Stage-7 - Stage-1 depends on stages: Stage-3 - Stage-7 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t1_n28 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: t2_n19 - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Forward - Statistics: Num rows: 11 Data size: 975 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t7_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: c1, cnt - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Group By Operator - aggregations: count(1) - keys: KEY._col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t8_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: int) - outputColumnNames: c1, cnt - Statistics: Num rows: 6 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(cnt, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t7_n1 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, cnt - Column Types: string, int - Table: default.t7_n1 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: c1, cnt - Column Types: string, int - Table: default.t8_n0 - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.t8_n0 - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, cnt from t2_n19 -) x -insert overwrite table t7_n1 - select c1, count(1) group by c1 -insert overwrite table t8_n0 - select c1, count(1) group by c1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n28 -PREHOOK: Input: default@t2_n19 -PREHOOK: Output: default@t7_n1 -PREHOOK: Output: default@t8_n0 -POSTHOOK: query: from -( - select key as c1, count(1) as cnt from t1_n28 group by key - union all - select key as c1, cnt from t2_n19 -) x -insert overwrite table t7_n1 - select c1, count(1) group by c1 -insert overwrite table t8_n0 - select c1, count(1) group by c1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n28 -POSTHOOK: Input: default@t2_n19 -POSTHOOK: Output: default@t7_n1 -POSTHOOK: Output: default@t8_n0 -POSTHOOK: Lineage: t7_n1.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t7_n1.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -POSTHOOK: Lineage: t8_n0.c1 EXPRESSION [(t1_n28)t1_n28.FieldSchema(name:key, type:string, comment:null), (t2_n19)t2_n19.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: t8_n0.cnt EXPRESSION [(t1_n28)t1_n28.null, (t2_n19)t2_n19.null, ] -PREHOOK: query: select * from t7_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@t7_n1 -#### A masked pattern was here #### -POSTHOOK: query: select * from t7_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t7_n1 -#### A masked pattern was here #### -0 2 -2 2 -4 2 -5 2 -8 2 -9 2 -PREHOOK: query: select * from t8_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@t8_n0 -#### A masked pattern was here #### -POSTHOOK: query: select * from t8_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t8_n0 -#### A masked pattern was here #### -0 2 -2 2 -4 2 -5 2 -8 2 -9 2 diff --git ql/src/test/results/clientpositive/union32.q.out ql/src/test/results/clientpositive/union32.q.out deleted file mode 100644 index b993d6030d..0000000000 --- ql/src/test/results/clientpositive/union32.q.out +++ /dev/null @@ -1,791 +0,0 @@ -PREHOOK: query: CREATE TABLE t1_n70 AS SELECT * FROM src WHERE key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_n70 -POSTHOOK: query: CREATE TABLE t1_n70 AS SELECT * FROM src WHERE key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_n70 -POSTHOOK: Lineage: t1_n70.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t1_n70.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE t2_n43 AS SELECT * FROM src WHERE key < 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@t2_n43 -POSTHOOK: query: CREATE TABLE t2_n43 AS SELECT * FROM src WHERE key < 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2_n43 -POSTHOOK: Lineage: t2_n43.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: t2_n43.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 -UNION ALL -SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 -UNION ALL -SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t1_n70 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: t2_n43 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(UDFToLong(key)) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 -UNION ALL -SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t1_n70 -UNION ALL -SELECT CAST(key AS BIGINT) AS key FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -2.0 -2.0 -4.0 -4.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -8.0 -8.0 -9.0 -9.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToLong(key) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(_col1) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: t2_n43 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -2.0 -2.0 -4.0 -4.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -8.0 -8.0 -9.0 -9.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToLong(key) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(_col1) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t2_n43 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(key) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -2.0 -2.0 -4.0 -4.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -5.0 -8.0 -8.0 -9.0 -9.0 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToLong(key) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), CAST( key AS CHAR(20)) (type: char(20)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(20)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 16 Data size: 1792 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(_col1) (type: double), CAST( _col3 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3072 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: t2_n43 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(key) (type: double), key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key -UNION ALL -SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -2.0 2 -2.0 2 -4.0 4 -4.0 4 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -8.0 8 -8.0 8 -9.0 9 -9.0 9 -PREHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS CHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToLong(key) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), CAST( key AS CHAR(20)) (type: char(20)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1890 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(20)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 16 Data size: 1792 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(_col1) (type: double), CAST( _col3 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3072 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t2_n43 - Statistics: Num rows: 10 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToDouble(key) (type: double), key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4002 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_n70 -PREHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM -(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2_n43 -UNION ALL -SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS VARCHAR(20)) AS value FROM t1_n70 a JOIN t2_n43 b ON a.key = b.key) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_n70 -POSTHOOK: Input: default@t2_n43 -#### A masked pattern was here #### -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -0.0 0 -2.0 2 -2.0 2 -4.0 4 -4.0 4 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -5.0 5 -8.0 8 -8.0 8 -9.0 9 -9.0 9 diff --git ql/src/test/results/clientpositive/union33.q.out ql/src/test/results/clientpositive/union33.q.out deleted file mode 100644 index f49201db5b..0000000000 --- ql/src/test/results/clientpositive/union33.q.out +++ /dev/null @@ -1,554 +0,0 @@ -PREHOOK: query: CREATE TABLE test_src_n1 (key STRING, value STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_src_n1 -POSTHOOK: query: CREATE TABLE test_src_n1 (key STRING, value STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_src_n1 -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION ALL - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_src_n1 -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION ALL - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src_n1 -STAGE DEPENDENCIES: - Stage-9 is a root stage - Stage-10 depends on stages: Stage-9 - Stage-2 depends on stages: Stage-10 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - -STAGE PLANS: - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: rand() (type: double) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: partials - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: final - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 0.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 0.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - Union - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: final - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.test_src_n1 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION ALL - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_src_n1 -POSTHOOK: query: INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, value FROM src - WHERE key = 0 -UNION ALL - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -)a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src_n1 -POSTHOOK: Lineage: test_src_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_src_n1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] -PREHOOK: query: SELECT COUNT(*) FROM test_src_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_src_n1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT COUNT(*) FROM test_src_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_src_n1 -#### A masked pattern was here #### -312 -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -UNION ALL - SELECT key, value FROM src - WHERE key = 0 -)a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_src_n1 -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -UNION ALL - SELECT key, value FROM src - WHERE key = 0 -)a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 - Stage-6 - Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 - Stage-4 depends on stages: Stage-0 - Stage-5 - Stage-7 - Stage-8 depends on stages: Stage-7 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: rand() (type: double) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: partials - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: final - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), CAST( _col1 AS STRING) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 0.0D) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 0.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 112250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: final - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-9 - Conditional Operator - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-4 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.test_src_n1 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_src_n1 - - Stage: Stage-8 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -UNION ALL - SELECT key, value FROM src - WHERE key = 0 -)a -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_src_n1 -POSTHOOK: query: INSERT OVERWRITE TABLE test_src_n1 -SELECT key, value FROM ( - SELECT key, cast(COUNT(*) as string) AS value FROM src - GROUP BY key -UNION ALL - SELECT key, value FROM src - WHERE key = 0 -)a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_src_n1 -POSTHOOK: Lineage: test_src_n1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_src_n1.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT COUNT(*) FROM test_src_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_src_n1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT COUNT(*) FROM test_src_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_src_n1 -#### A masked pattern was here #### -312 diff --git ql/src/test/results/clientpositive/union34.q.out ql/src/test/results/clientpositive/union34.q.out deleted file mode 100644 index 9c296ed346..0000000000 --- ql/src/test/results/clientpositive/union34.q.out +++ /dev/null @@ -1,428 +0,0 @@ -PREHOOK: query: create table src10_1_n0 (key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src10_1_n0 -POSTHOOK: query: create table src10_1_n0 (key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_1_n0 -PREHOOK: query: create table src10_2_n0 (key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src10_2_n0 -POSTHOOK: query: create table src10_2_n0 (key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_2_n0 -PREHOOK: query: create table src10_3_n0 (key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src10_3_n0 -POSTHOOK: query: create table src10_3_n0 (key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_3_n0 -PREHOOK: query: create table src10_4_n0 (key string, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src10_4_n0 -POSTHOOK: query: create table src10_4_n0 (key string, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src10_4_n0 -PREHOOK: query: from (select * from src tablesample (10 rows)) a -insert overwrite table src10_1_n0 select * -insert overwrite table src10_2_n0 select * -insert overwrite table src10_3_n0 select * -insert overwrite table src10_4_n0 select * -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@src10_1_n0 -PREHOOK: Output: default@src10_2_n0 -PREHOOK: Output: default@src10_3_n0 -PREHOOK: Output: default@src10_4_n0 -POSTHOOK: query: from (select * from src tablesample (10 rows)) a -insert overwrite table src10_1_n0 select * -insert overwrite table src10_2_n0 select * -insert overwrite table src10_3_n0 select * -insert overwrite table src10_4_n0 select * -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@src10_1_n0 -POSTHOOK: Output: default@src10_2_n0 -POSTHOOK: Output: default@src10_3_n0 -POSTHOOK: Output: default@src10_4_n0 -POSTHOOK: Lineage: src10_1_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_1_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_2_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_2_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_3_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_3_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: src10_4_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src10_4_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1_n0 -PREHOOK: Input: default@src10_2_n0 -PREHOOK: Input: default@src10_3_n0 -PREHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -POSTHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1_n0 -POSTHOOK: Input: default@src10_2_n0 -POSTHOOK: Input: default@src10_3_n0 -POSTHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-2 depends on stages: Stage-7 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-7 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1-subquery1-subquery1:$hdt$_0:src10_1_n0 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1-subquery1-subquery1:$hdt$_0:src10_1_n0 - TableScan - alias: src10_1_n0 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src10_3_n0 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src10_4_n0 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src10_2_n0 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1_n0 -PREHOOK: Input: default@src10_2_n0 -PREHOOK: Input: default@src10_3_n0 -PREHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1_n0 -POSTHOOK: Input: default@src10_2_n0 -POSTHOOK: Input: default@src10_3_n0 -POSTHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -165 val_165 -165 val_165 -165 val_165 -238 val_238 -238 val_238 -238 val_238 -255 val_255 -255 val_255 -255 val_255 -27 val_27 -27 val_27 -27 val_27 -278 val_278 -278 val_278 -278 val_278 -311 val_311 -311 val_311 -311 val_311 -409 val_409 -409 val_409 -409 val_409 -484 val_484 -484 val_484 -484 val_484 -86 val_86 -86 val_86 -86 val_86 -98 val_98 -98 val_98 -98 val_98 -PREHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1_n0 -PREHOOK: Input: default@src10_2_n0 -PREHOOK: Input: default@src10_3_n0 -PREHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -POSTHOOK: query: explain -SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1_n0 -POSTHOOK: Input: default@src10_2_n0 -POSTHOOK: Input: default@src10_3_n0 -POSTHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src10_1_n0 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - TableScan - alias: src10_2_n0 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src10_3_n0 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src10_4_n0 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 30 Data size: 5340 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src10_1_n0 -PREHOOK: Input: default@src10_2_n0 -PREHOOK: Input: default@src10_3_n0 -PREHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM ( - SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1_n0) sub1 JOIN (SELECT * FROM src10_2_n0) sub0 ON (sub0.key = sub1.key) - UNION ALL - SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3_n0) sub2 UNION ALL SELECT * FROM src10_4_n0 ) alias0 -) alias1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src10_1_n0 -POSTHOOK: Input: default@src10_2_n0 -POSTHOOK: Input: default@src10_3_n0 -POSTHOOK: Input: default@src10_4_n0 -#### A masked pattern was here #### -165 val_165 -165 val_165 -165 val_165 -238 val_238 -238 val_238 -238 val_238 -255 val_255 -255 val_255 -255 val_255 -27 val_27 -27 val_27 -27 val_27 -278 val_278 -278 val_278 -278 val_278 -311 val_311 -311 val_311 -311 val_311 -409 val_409 -409 val_409 -409 val_409 -484 val_484 -484 val_484 -484 val_484 -86 val_86 -86 val_86 -86 val_86 -98 val_98 -98 val_98 -98 val_98 diff --git ql/src/test/results/clientpositive/union37.q.out ql/src/test/results/clientpositive/union37.q.out deleted file mode 100644 index 97e0a4e883..0000000000 --- ql/src/test/results/clientpositive/union37.q.out +++ /dev/null @@ -1,546 +0,0 @@ -PREHOOK: query: create table l_test1 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@l_test1 -POSTHOOK: query: create table l_test1 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@l_test1 -PREHOOK: query: insert into l_test1 values (1, "table_1", "2016-08-11") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@l_test1 -POSTHOOK: query: insert into l_test1 values (1, "table_1", "2016-08-11") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@l_test1 -POSTHOOK: Lineage: l_test1.id SCRIPT [] -POSTHOOK: Lineage: l_test1.trans_date SCRIPT [] -POSTHOOK: Lineage: l_test1.val SCRIPT [] -PREHOOK: query: create table l_test2 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@l_test2 -POSTHOOK: query: create table l_test2 (id bigint,val string,trans_date string) row format delimited fields terminated by ' ' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@l_test2 -PREHOOK: query: insert into l_test2 values (2, "table_2", "2016-08-11") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@l_test2 -POSTHOOK: query: insert into l_test2 values (2, "table_2", "2016-08-11") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@l_test2 -POSTHOOK: Lineage: l_test2.id SCRIPT [] -POSTHOOK: Lineage: l_test2.trans_date SCRIPT [] -POSTHOOK: Lineage: l_test2.val SCRIPT [] -PREHOOK: query: explain -select - id, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: explain -select - id, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: l_test1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: l_test2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), val (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select - id, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: select - id, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -1 table_1 2016-08-11 -2 table_2 2016-08-11 -PREHOOK: query: explain -select - id, - 999, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: explain -select - id, - 999, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: l_test1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: l_test2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), val (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 394 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select - id, - 999, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: select - id, - 999, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -1 999 table_1 2016-08-11 -2 999 table_2 2016-08-11 -PREHOOK: query: explain -select - id, - 999, - 666, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - 666, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: explain -select - id, - 999, - 666, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - 666, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: l_test1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), 666 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: l_test2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), val (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 386 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), 666 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select - id, - 999, - 666, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - 666, - val, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: select - id, - 999, - 666, - 'table_1' , - trans_date -from l_test1 -union all -select - id, - 999, - 666, - val, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -1 999 666 table_1 2016-08-11 -2 999 666 table_2 2016-08-11 -PREHOOK: query: explain -select - id, - 999, - 'table_1' , - trans_date, - '2016-11-11' -from l_test1 -union all -select - id, - 999, - val, - trans_date, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: explain -select - id, - 999, - 'table_1' , - trans_date, - '2016-11-11' -from l_test1 -union all -select - id, - 999, - val, - trans_date, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: l_test1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), 'table_1' (type: string), trans_date (type: string), '2016-11-11' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 287 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 574 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: l_test2 - Statistics: Num rows: 1 Data size: 193 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: bigint), val (type: string), trans_date (type: string), trans_date (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 287 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 574 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), 999 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select - id, - 999, - 'table_1' , - trans_date, - '2016-11-11' -from l_test1 -union all -select - id, - 999, - val, - trans_date, - trans_date -from l_test2 -PREHOOK: type: QUERY -PREHOOK: Input: default@l_test1 -PREHOOK: Input: default@l_test2 -#### A masked pattern was here #### -POSTHOOK: query: select - id, - 999, - 'table_1' , - trans_date, - '2016-11-11' -from l_test1 -union all -select - id, - 999, - val, - trans_date, - trans_date -from l_test2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@l_test1 -POSTHOOK: Input: default@l_test2 -#### A masked pattern was here #### -1 999 table_1 2016-08-11 2016-11-11 -2 999 table_2 2016-08-11 2016-08-11 diff --git ql/src/test/results/clientpositive/union_lateralview.q.out ql/src/test/results/clientpositive/union_lateralview.q.out deleted file mode 100644 index 06dae33e7e..0000000000 --- ql/src/test/results/clientpositive/union_lateralview.q.out +++ /dev/null @@ -1,329 +0,0 @@ -PREHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_union_lateral_view -POSTHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_union_lateral_view -PREHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE test_union_lateral_view -SELECT b.key, d.arr_ele, d.value -FROM ( - SELECT c.arr_ele as arr_ele, a.key as key, a.value as value - FROM ( - SELECT key, value, array(1,2,3) as arr - FROM src - - UNION ALL - - SELECT key, value, array(1,2,3) as arr - FROM srcpart - WHERE ds = '2008-04-08' and hr='12' - ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele -) d -LEFT OUTER JOIN src b -ON d.key = b.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Output: default@test_union_lateral_view -POSTHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE test_union_lateral_view -SELECT b.key, d.arr_ele, d.value -FROM ( - SELECT c.arr_ele as arr_ele, a.key as key, a.value as value - FROM ( - SELECT key, value, array(1,2,3) as arr - FROM src - - UNION ALL - - SELECT key, value, array(1,2,3) as arr - FROM srcpart - WHERE ds = '2008-04-08' and hr='12' - ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele -) d -LEFT OUTER JOIN src b -ON d.key = b.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@test_union_lateral_view -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), array(1,2,3) (type: array) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 117000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 290000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - Select Operator - expressions: _col2 (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - TableScan - alias: srcpart - filterExpr: ((ds = '2008-04-08') and (hr = '12')) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), array(1,2,3) (type: array) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 117000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Forward - Statistics: Num rows: 1000 Data size: 234000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 290000 Basic stats: COMPLETE Column stats: COMPLETE - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - Select Operator - expressions: _col2 (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 1000 Data size: 56000 Basic stats: COMPLETE Column stats: COMPLETE - function name: explode - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 2000 Data size: 346000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2000 Data size: 462000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 3104 Data size: 468967 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col3) (type: int), _col0 (type: int), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_union_lateral_view - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - outputColumnNames: key, arr_ele, value - Statistics: Num rows: 3104 Data size: 294880 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(arr_ele, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_union_lateral_view - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, arr_ele, value - Column Types: int, int, string - Table: default.test_union_lateral_view - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view -SELECT b.key, d.arr_ele, d.value -FROM ( - SELECT c.arr_ele as arr_ele, a.key as key, a.value as value - FROM ( - SELECT key, value, array(1,2,3) as arr - FROM src - - UNION ALL - - SELECT key, value, array(1,2,3) as arr - FROM srcpart - WHERE ds = '2008-04-08' and hr='12' - ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele -) d -LEFT OUTER JOIN src b -ON d.key = b.key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Output: default@test_union_lateral_view -POSTHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view -SELECT b.key, d.arr_ele, d.value -FROM ( - SELECT c.arr_ele as arr_ele, a.key as key, a.value as value - FROM ( - SELECT key, value, array(1,2,3) as arr - FROM src - - UNION ALL - - SELECT key, value, array(1,2,3) as arr - FROM srcpart - WHERE ds = '2008-04-08' and hr='12' - ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele -) d -LEFT OUTER JOIN src b -ON d.key = b.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@test_union_lateral_view -POSTHOOK: Lineage: test_union_lateral_view.arr_ele SCRIPT [] -POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_union_lateral_view -#### A masked pattern was here #### -POSTHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_union_lateral_view -#### A masked pattern was here #### -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 1 val_0 -0 2 val_0 -0 2 val_0 diff --git ql/src/test/results/clientpositive/union_offcbo.q.out ql/src/test/results/clientpositive/union_offcbo.q.out deleted file mode 100644 index 16afd016c2..0000000000 --- ql/src/test/results/clientpositive/union_offcbo.q.out +++ /dev/null @@ -1,1863 +0,0 @@ -PREHOOK: query: DROP TABLE IF EXISTS ttest1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS ttest1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE IF EXISTS ttest2 -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS ttest2 -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE ttest1 ( - `id1` bigint COMMENT 'from deserializer', - `ts1` string COMMENT 'from deserializer', - `dt1` string COMMENT 'from deserializer', - `dt2` string COMMENT 'from deserializer', - `ac1` string COMMENT 'from deserializer', - `kd1` string COMMENT 'from deserializer', - `sts` string COMMENT 'from deserializer', - `at1` bigint COMMENT 'from deserializer') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ttest1 -POSTHOOK: query: CREATE TABLE ttest1 ( - `id1` bigint COMMENT 'from deserializer', - `ts1` string COMMENT 'from deserializer', - `dt1` string COMMENT 'from deserializer', - `dt2` string COMMENT 'from deserializer', - `ac1` string COMMENT 'from deserializer', - `kd1` string COMMENT 'from deserializer', - `sts` string COMMENT 'from deserializer', - `at1` bigint COMMENT 'from deserializer') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ttest1 -PREHOOK: query: CREATE TABLE ttest2 ( - `id1` bigint, - `ts1` string, - `dt1` string, - `dt2` string, - `ac1` string, - `kd1` string, - `sts` string, - `at1` bigint, - `khash` string, - `rhash` string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ttest2 -POSTHOOK: query: CREATE TABLE ttest2 ( - `id1` bigint, - `ts1` string, - `dt1` string, - `dt2` string, - `ac1` string, - `kd1` string, - `sts` string, - `at1` bigint, - `khash` string, - `rhash` string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ttest2 -PREHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -PREHOOK: type: QUERY -PREHOOK: Input: default@ttest1 -PREHOOK: Input: default@ttest2 -#### A masked pattern was here #### -POSTHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ttest1 -POSTHOOK: Input: default@ttest2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col8, _col9 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - value expressions: _col9 (type: string) - TableScan - alias: ttest2 - filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and ((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - TableScan - alias: ttest2 - filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ts1 (type: string), khash (type: string), rhash (type: string) - outputColumnNames: _col1, _col8, _col9 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END <> 'RET') and ((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -PREHOOK: type: QUERY -PREHOOK: Input: default@ttest1 -PREHOOK: Input: default@ttest2 -#### A masked pattern was here #### -POSTHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ttest1 -POSTHOOK: Input: default@ttest2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - TableScan - alias: ttest2 - filterExpr: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col5 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint), _col6 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 is null and (_col3 >= '2016-02-05')) or (_col0 is null and (_col3 <= '2016-02-05')) is not true) and ((COALESCE(_col0,-1) <> COALESCE(_col7,-1)) or (COALESCE(_col1,-1) <> COALESCE(_col8,-1)))) (type: boolean) - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col3 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) - TableScan - alias: ttest2 - filterExpr: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ts1 (type: string), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((COALESCE(_col3,-1) <> COALESCE(_col6,-1)) or (COALESCE(_col4,-1) <> COALESCE(_col7,-1))) (type: boolean) - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -PREHOOK: type: QUERY -PREHOOK: Input: default@ttest1 -PREHOOK: Input: default@ttest2 -#### A masked pattern was here #### -POSTHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ttest1 -POSTHOOK: Input: default@ttest2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col8, _col9 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - value expressions: _col9 (type: string) - TableScan - alias: ttest2 - filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - filterExpr: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - TableScan - alias: ttest2 - filterExpr: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ts1 (type: string), khash (type: string), rhash (type: string) - outputColumnNames: _col1, _col8, _col9 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -PREHOOK: type: QUERY -PREHOOK: Input: default@ttest1 -PREHOOK: Input: default@ttest2 -#### A masked pattern was here #### -POSTHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ttest1 -POSTHOOK: Input: default@ttest2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col8, _col9 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: NONE - value expressions: _col9 (type: string) - TableScan - alias: ttest2 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col2, _col3, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col18 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col7 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 444 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9 - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 294 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col6 (type: string), _col7 (type: bigint), _col9 (type: string) - TableScan - alias: ttest2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: '2015-11-20' BETWEEN dt1 AND dt2 (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ts1 (type: string), khash (type: string), rhash (type: string) - outputColumnNames: _col1, _col8, _col9 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col9 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col8 (type: string) - 1 _col8 (type: string) - outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((COALESCE(_col8,-1) <> COALESCE(_col18,-1)) or (COALESCE(_col9,-1) <> COALESCE(_col19,-1))) and _col8 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -PREHOOK: type: QUERY -PREHOOK: Input: default@ttest1 -PREHOOK: Input: default@ttest2 -#### A masked pattern was here #### -POSTHOOK: query: explain SELECT - A2.id1, A2.sts,A2.at1, - CASE WHEN FLAG = 'A_INS' THEN cast(date_add('2015-11-20', 1) as string) ELSE '2015-11-20' END dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,A2.FLAG - FROM ( - SELECT - A2.id1, A2.sts,A2.at1 - ,A2.dt1 - ,A2.dt2 - ,A2.khash - ,A2.rhash - ,CASE - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 >= '2016-02-05' - THEN 'DEL' - - WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.dt1 <= '2016-02-05' - THEN 'RET' - - WHEN - ( - A2.khash = A1.khash - AND A2.rhash <> A1.rhash - ) - THEN 'A_INS' - - ELSE 'NA' - END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 AND dt2 - ) A2 - ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A2.khash IS NOT NULL - - UNION ALL - - SELECT A1.id1, A1.sts,A1.at1 - ,A1.dt1 - , '2099-12-31' dt2 - ,A1.khash - ,A1.rhash - ,CASE WHEN A2.khash IS NOT NULL - AND A1.khash IS NULL - AND A2.ts1 <= A1.ts1 - THEN 'DEL' - - WHEN ( A2.khash IS NULL AND A1.khash IS NOT NULL ) - OR ( A2.khash = A1.khash AND A2.rhash <> A1.rhash ) THEN 'INS' ELSE 'NA' END FLAG - FROM ( - SELECT * - ,reflect('org.apache.commons.codec.digest.DigestUtils', 'sha256Hex',concat(id1)) khash - ,reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) rhash - FROM ttest1 - WHERE ts1 = '2015-11-20' - ) A1 - FULL OUTER JOIN ( - SELECT * - FROM ttest2 - WHERE '2015-11-20' BETWEEN dt1 - AND dt2 - ) A2 ON A1.khash = A2.khash - WHERE NOT ( - NVL(A1.khash, - 1) = NVL(A2.khash, - 1) - AND NVL(A1.rhash, - 1) = NVL(A2.rhash, - 1) - ) - AND A1.khash IS NOT NULL - ) A2 - where a2.flag <> 'RET' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ttest1 -POSTHOOK: Input: default@ttest2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - TableScan - alias: ttest2 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), dt1 (type: string), dt2 (type: string), sts (type: string), at1 (type: bigint), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col5 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col5 (type: string) - Statistics: Num rows: 1 Data size: 936 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint), _col6 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 is null and (_col3 >= '2016-02-05')) or (_col0 is null and (_col3 <= '2016-02-05')) is not true) and ((COALESCE(_col0,-1) <> COALESCE(_col7,-1)) or (COALESCE(_col1,-1) <> COALESCE(_col8,-1)))) (type: boolean) - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - Union - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), CASE WHEN ((_col6 = 'A_INS')) THEN ('2015-11-21') ELSE ('2015-11-20') END (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 642 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: ttest1 - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ts1 = '2015-11-20') (type: boolean) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( id1 AS STRING))) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(CAST( at1 AS STRING))) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col3 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) - TableScan - alias: ttest2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('2015-11-20' BETWEEN dt1 AND dt2 and khash is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ts1 (type: string), khash (type: string), rhash (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((COALESCE(_col3,-1) <> COALESCE(_col6,-1)) or (COALESCE(_col4,-1) <> COALESCE(_col7,-1))) (type: boolean) - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 1 Data size: 422 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: DROP TABLE ttest1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@ttest1 -PREHOOK: Output: default@ttest1 -POSTHOOK: query: DROP TABLE ttest1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@ttest1 -POSTHOOK: Output: default@ttest1 -PREHOOK: query: DROP TABLE ttest2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@ttest2 -PREHOOK: Output: default@ttest2 -POSTHOOK: query: DROP TABLE ttest2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@ttest2 -POSTHOOK: Output: default@ttest2 diff --git ql/src/test/results/clientpositive/union_pos_alias.q.out ql/src/test/results/clientpositive/union_pos_alias.q.out deleted file mode 100644 index 2b48b75cae..0000000000 --- ql/src/test/results/clientpositive/union_pos_alias.q.out +++ /dev/null @@ -1,658 +0,0 @@ -PREHOOK: query: explain -select 'tst1' as key, count(1) as value from src s1 -UNION ALL -select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 -order by 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain -select 'tst1' as key, count(1) as value from src s1 -UNION ALL -select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 -order by 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 - Stage-3 is a root stage - Stage-4 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst1' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst2' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'tst3' (type: string), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select 'tst1' as key, count(1) as value from src s1 -UNION ALL -select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 -order by 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select 'tst1' as key, count(1) as value from src s1 -UNION ALL -select key, value from (select 'tst2' as key, count(1) as value from src s2 UNION ALL select 'tst3' as key, count(1) as value from src s3) s4 -order by 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -tst1 500 -tst2 500 -tst3 500 -PREHOOK: query: drop table src_10 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table src_10 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table src_10 as select * from src limit 10 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@src_10 -POSTHOOK: query: create table src_10 as select * from src limit 10 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_10 -POSTHOOK: Lineage: src_10.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: src_10.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain -select key as value, value as key from src_10 -UNION ALL -select 'test', value from src_10 s3 -order by 2, 1 desc -PREHOOK: type: QUERY -PREHOOK: Input: default@src_10 -#### A masked pattern was here #### -POSTHOOK: query: explain -select key as value, value as key from src_10 -UNION ALL -select 'test', value from src_10 s3 -order by 2, 1 desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_10 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 3570 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - null sort order: zz - sort order: +- - Statistics: Num rows: 20 Data size: 3570 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: s3 - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'test' (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1790 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 20 Data size: 3570 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - null sort order: zz - sort order: +- - Statistics: Num rows: 20 Data size: 3570 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 3580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select key as value, value as key from src_10 -UNION ALL -select 'test', value from src_10 s3 -order by 2, 1 desc -PREHOOK: type: QUERY -PREHOOK: Input: default@src_10 -#### A masked pattern was here #### -POSTHOOK: query: select key as value, value as key from src_10 -UNION ALL -select 'test', value from src_10 s3 -order by 2, 1 desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_10 -#### A masked pattern was here #### -test val_165 -165 val_165 -test val_238 -238 val_238 -test val_255 -255 val_255 -test val_27 -27 val_27 -test val_278 -278 val_278 -test val_311 -311 val_311 -test val_409 -409 val_409 -test val_484 -484 val_484 -test val_86 -86 val_86 -test val_98 -98 val_98 -PREHOOK: query: drop table src_10 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@src_10 -PREHOOK: Output: default@src_10 -POSTHOOK: query: drop table src_10 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@src_10 -POSTHOOK: Output: default@src_10 -PREHOOK: query: drop view v_n8 -PREHOOK: type: DROPVIEW -POSTHOOK: query: drop view v_n8 -POSTHOOK: type: DROPVIEW -PREHOOK: query: create view v_n8 as select key as k from src intersect all select key as k1 from src -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@v_n8 -POSTHOOK: query: create view v_n8 as select key as k from src intersect all select key as k1 from src -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v_n8 -POSTHOOK: Lineage: v_n8.k SCRIPT [(src)src.null, (src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: desc formatted v_n8 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v_n8 -POSTHOOK: query: desc formatted v_n8 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v_n8 -# col_name data_type comment -k string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -Table Type: VIRTUAL_VIEW -Table Parameters: - bucketing_version 2 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: null -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] - -# View Information -Original Query: select key as k from src intersect all select key as k1 from src -Expanded Query: select `src`.`key` as `k` from `default`.`src` intersect all select `src`.`key` as `k1` from `default`.`src` -PREHOOK: query: create table masking_test_n9 as select cast(key as int) as key, value from src -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@masking_test_n9 -POSTHOOK: query: create table masking_test_n9 as select cast(key as int) as key, value from src -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@masking_test_n9 -POSTHOOK: Lineage: masking_test_n9.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: masking_test_n9.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: explain -select * from masking_test_n9 union all select * from masking_test_n9 -PREHOOK: type: QUERY -PREHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -POSTHOOK: query: explain -select * from masking_test_n9 union all select * from masking_test_n9 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: masking_test_n9 - filterExpr: ((key < 10) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: masking_test_n9 - filterExpr: ((key < 10) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 940 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 1880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select * from masking_test_n9 union all select * from masking_test_n9 -PREHOOK: type: QUERY -PREHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -POSTHOOK: query: select * from masking_test_n9 union all select * from masking_test_n9 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -0 0_lav -0 0_lav -4 4_lav -4 4_lav -8 8_lav -8 8_lav -0 0_lav -0 0_lav -0 0_lav -0 0_lav -2 2_lav -2 2_lav -PREHOOK: query: explain -select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 -PREHOOK: type: QUERY -PREHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -POSTHOOK: query: explain -select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: masking_test_n9 - filterExpr: ((key < 10) and (key > 0) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 0) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: int), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 4 Data size: 784 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col2), count(_col2) - keys: _col0 (type: int), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - TableScan - Union - Statistics: Num rows: 4 Data size: 784 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col2), count(_col2) - keys: _col0 (type: int), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), count(VALUE._col1) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col3 = 2L) (type: boolean) - Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - function name: UDTFReplicateRows - Select Operator - expressions: col1 (type: int), col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: masking_test_n9 - filterExpr: ((key < 10) and (key > 0) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 0) and ((key % 2) = 0)) (type: boolean) - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 475 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: int), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 -PREHOOK: type: QUERY -PREHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -POSTHOOK: query: select key as k1, value as v1 from masking_test_n9 where key > 0 intersect all select key as k2, value as v2 from masking_test_n9 where key > 0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@masking_test_n9 -#### A masked pattern was here #### -2 2_lav -4 4_lav -8 8_lav diff --git ql/src/test/results/clientpositive/union_ppr.q.out ql/src/test/results/clientpositive/union_ppr.q.out deleted file mode 100644 index b841994373..0000000000 --- ql/src/test/results/clientpositive/union_ppr.q.out +++ /dev/null @@ -1,591 +0,0 @@ -PREHOOK: query: EXPLAIN EXTENDED -SELECT * FROM ( - SELECT X.* FROM SRCPART X WHERE X.key < 100 - UNION ALL - SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 -) A -WHERE A.ds = '2008-04-08' -SORT BY A.key, A.value, A.ds, A.hr -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED -SELECT * FROM ( - SELECT X.* FROM SRCPART X WHERE X.key < 100 - UNION ALL - SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 -) A -WHERE A.ds = '2008-04-08' -SORT BY A.key, A.value, A.ds, A.hr -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: x - filterExpr: ((UDFToDouble(key) < 100.0D) and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 100.0D) (type: boolean) - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 666 Data size: 241092 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - null sort order: zzz - numBuckets: -1 - sort order: +++ - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - auto parallelism: false - TableScan - alias: y - filterExpr: ((UDFToDouble(key) < 100.0D) and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (UDFToDouble(key) < 100.0D) (type: boolean) - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 666 Data size: 241092 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - null sort order: zzz - numBuckets: -1 - sort order: +++ - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [$hdt$_0-subquery1:x, $hdt$_0-subquery2:y] - /srcpart/ds=2008-04-08/hr=12 [$hdt$_0-subquery1:x, $hdt$_0-subquery2:y] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 666 Data size: 303696 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * FROM ( - SELECT X.* FROM SRCPART X WHERE X.key < 100 - UNION ALL - SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 -) A -WHERE A.ds = '2008-04-08' -SORT BY A.key, A.value, A.ds, A.hr -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM ( - SELECT X.* FROM SRCPART X WHERE X.key < 100 - UNION ALL - SELECT Y.* FROM SRCPART Y WHERE Y.key < 100 -) A -WHERE A.ds = '2008-04-08' -SORT BY A.key, A.value, A.ds, A.hr -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 11 -0 val_0 2008-04-08 12 -0 val_0 2008-04-08 12 -0 val_0 2008-04-08 12 -0 val_0 2008-04-08 12 -0 val_0 2008-04-08 12 -0 val_0 2008-04-08 12 -10 val_10 2008-04-08 11 -10 val_10 2008-04-08 11 -10 val_10 2008-04-08 12 -10 val_10 2008-04-08 12 -11 val_11 2008-04-08 11 -11 val_11 2008-04-08 11 -11 val_11 2008-04-08 12 -11 val_11 2008-04-08 12 -12 val_12 2008-04-08 11 -12 val_12 2008-04-08 11 -12 val_12 2008-04-08 11 -12 val_12 2008-04-08 11 -12 val_12 2008-04-08 12 -12 val_12 2008-04-08 12 -12 val_12 2008-04-08 12 -12 val_12 2008-04-08 12 -15 val_15 2008-04-08 11 -15 val_15 2008-04-08 11 -15 val_15 2008-04-08 11 -15 val_15 2008-04-08 11 -15 val_15 2008-04-08 12 -15 val_15 2008-04-08 12 -15 val_15 2008-04-08 12 -15 val_15 2008-04-08 12 -17 val_17 2008-04-08 11 -17 val_17 2008-04-08 11 -17 val_17 2008-04-08 12 -17 val_17 2008-04-08 12 -18 val_18 2008-04-08 11 -18 val_18 2008-04-08 11 -18 val_18 2008-04-08 11 -18 val_18 2008-04-08 11 -18 val_18 2008-04-08 12 -18 val_18 2008-04-08 12 -18 val_18 2008-04-08 12 -18 val_18 2008-04-08 12 -19 val_19 2008-04-08 11 -19 val_19 2008-04-08 11 -19 val_19 2008-04-08 12 -19 val_19 2008-04-08 12 -2 val_2 2008-04-08 11 -2 val_2 2008-04-08 11 -2 val_2 2008-04-08 12 -2 val_2 2008-04-08 12 -20 val_20 2008-04-08 11 -20 val_20 2008-04-08 11 -20 val_20 2008-04-08 12 -20 val_20 2008-04-08 12 -24 val_24 2008-04-08 11 -24 val_24 2008-04-08 11 -24 val_24 2008-04-08 11 -24 val_24 2008-04-08 11 -24 val_24 2008-04-08 12 -24 val_24 2008-04-08 12 -24 val_24 2008-04-08 12 -24 val_24 2008-04-08 12 -26 val_26 2008-04-08 11 -26 val_26 2008-04-08 11 -26 val_26 2008-04-08 11 -26 val_26 2008-04-08 11 -26 val_26 2008-04-08 12 -26 val_26 2008-04-08 12 -26 val_26 2008-04-08 12 -26 val_26 2008-04-08 12 -27 val_27 2008-04-08 11 -27 val_27 2008-04-08 11 -27 val_27 2008-04-08 12 -27 val_27 2008-04-08 12 -28 val_28 2008-04-08 11 -28 val_28 2008-04-08 11 -28 val_28 2008-04-08 12 -28 val_28 2008-04-08 12 -30 val_30 2008-04-08 11 -30 val_30 2008-04-08 11 -30 val_30 2008-04-08 12 -30 val_30 2008-04-08 12 -33 val_33 2008-04-08 11 -33 val_33 2008-04-08 11 -33 val_33 2008-04-08 12 -33 val_33 2008-04-08 12 -34 val_34 2008-04-08 11 -34 val_34 2008-04-08 11 -34 val_34 2008-04-08 12 -34 val_34 2008-04-08 12 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 11 -35 val_35 2008-04-08 12 -35 val_35 2008-04-08 12 -35 val_35 2008-04-08 12 -35 val_35 2008-04-08 12 -35 val_35 2008-04-08 12 -35 val_35 2008-04-08 12 -37 val_37 2008-04-08 11 -37 val_37 2008-04-08 11 -37 val_37 2008-04-08 11 -37 val_37 2008-04-08 11 -37 val_37 2008-04-08 12 -37 val_37 2008-04-08 12 -37 val_37 2008-04-08 12 -37 val_37 2008-04-08 12 -4 val_4 2008-04-08 11 -4 val_4 2008-04-08 11 -4 val_4 2008-04-08 12 -4 val_4 2008-04-08 12 -41 val_41 2008-04-08 11 -41 val_41 2008-04-08 11 -41 val_41 2008-04-08 12 -41 val_41 2008-04-08 12 -42 val_42 2008-04-08 11 -42 val_42 2008-04-08 11 -42 val_42 2008-04-08 11 -42 val_42 2008-04-08 11 -42 val_42 2008-04-08 12 -42 val_42 2008-04-08 12 -42 val_42 2008-04-08 12 -42 val_42 2008-04-08 12 -43 val_43 2008-04-08 11 -43 val_43 2008-04-08 11 -43 val_43 2008-04-08 12 -43 val_43 2008-04-08 12 -44 val_44 2008-04-08 11 -44 val_44 2008-04-08 11 -44 val_44 2008-04-08 12 -44 val_44 2008-04-08 12 -47 val_47 2008-04-08 11 -47 val_47 2008-04-08 11 -47 val_47 2008-04-08 12 -47 val_47 2008-04-08 12 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 -51 val_51 2008-04-08 11 -51 val_51 2008-04-08 11 -51 val_51 2008-04-08 11 -51 val_51 2008-04-08 11 -51 val_51 2008-04-08 12 -51 val_51 2008-04-08 12 -51 val_51 2008-04-08 12 -51 val_51 2008-04-08 12 -53 val_53 2008-04-08 11 -53 val_53 2008-04-08 11 -53 val_53 2008-04-08 12 -53 val_53 2008-04-08 12 -54 val_54 2008-04-08 11 -54 val_54 2008-04-08 11 -54 val_54 2008-04-08 12 -54 val_54 2008-04-08 12 -57 val_57 2008-04-08 11 -57 val_57 2008-04-08 11 -57 val_57 2008-04-08 12 -57 val_57 2008-04-08 12 -58 val_58 2008-04-08 11 -58 val_58 2008-04-08 11 -58 val_58 2008-04-08 11 -58 val_58 2008-04-08 11 -58 val_58 2008-04-08 12 -58 val_58 2008-04-08 12 -58 val_58 2008-04-08 12 -58 val_58 2008-04-08 12 -64 val_64 2008-04-08 11 -64 val_64 2008-04-08 11 -64 val_64 2008-04-08 12 -64 val_64 2008-04-08 12 -65 val_65 2008-04-08 11 -65 val_65 2008-04-08 11 -65 val_65 2008-04-08 12 -65 val_65 2008-04-08 12 -66 val_66 2008-04-08 11 -66 val_66 2008-04-08 11 -66 val_66 2008-04-08 12 -66 val_66 2008-04-08 12 -67 val_67 2008-04-08 11 -67 val_67 2008-04-08 11 -67 val_67 2008-04-08 11 -67 val_67 2008-04-08 11 -67 val_67 2008-04-08 12 -67 val_67 2008-04-08 12 -67 val_67 2008-04-08 12 -67 val_67 2008-04-08 12 -69 val_69 2008-04-08 11 -69 val_69 2008-04-08 11 -69 val_69 2008-04-08 12 -69 val_69 2008-04-08 12 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 11 -70 val_70 2008-04-08 12 -70 val_70 2008-04-08 12 -70 val_70 2008-04-08 12 -70 val_70 2008-04-08 12 -70 val_70 2008-04-08 12 -70 val_70 2008-04-08 12 -72 val_72 2008-04-08 11 -72 val_72 2008-04-08 11 -72 val_72 2008-04-08 11 -72 val_72 2008-04-08 11 -72 val_72 2008-04-08 12 -72 val_72 2008-04-08 12 -72 val_72 2008-04-08 12 -72 val_72 2008-04-08 12 -74 val_74 2008-04-08 11 -74 val_74 2008-04-08 11 -74 val_74 2008-04-08 12 -74 val_74 2008-04-08 12 -76 val_76 2008-04-08 11 -76 val_76 2008-04-08 11 -76 val_76 2008-04-08 11 -76 val_76 2008-04-08 11 -76 val_76 2008-04-08 12 -76 val_76 2008-04-08 12 -76 val_76 2008-04-08 12 -76 val_76 2008-04-08 12 -77 val_77 2008-04-08 11 -77 val_77 2008-04-08 11 -77 val_77 2008-04-08 12 -77 val_77 2008-04-08 12 -78 val_78 2008-04-08 11 -78 val_78 2008-04-08 11 -78 val_78 2008-04-08 12 -78 val_78 2008-04-08 12 -8 val_8 2008-04-08 11 -8 val_8 2008-04-08 11 -8 val_8 2008-04-08 12 -8 val_8 2008-04-08 12 -80 val_80 2008-04-08 11 -80 val_80 2008-04-08 11 -80 val_80 2008-04-08 12 -80 val_80 2008-04-08 12 -82 val_82 2008-04-08 11 -82 val_82 2008-04-08 11 -82 val_82 2008-04-08 12 -82 val_82 2008-04-08 12 -83 val_83 2008-04-08 11 -83 val_83 2008-04-08 11 -83 val_83 2008-04-08 11 -83 val_83 2008-04-08 11 -83 val_83 2008-04-08 12 -83 val_83 2008-04-08 12 -83 val_83 2008-04-08 12 -83 val_83 2008-04-08 12 -84 val_84 2008-04-08 11 -84 val_84 2008-04-08 11 -84 val_84 2008-04-08 11 -84 val_84 2008-04-08 11 -84 val_84 2008-04-08 12 -84 val_84 2008-04-08 12 -84 val_84 2008-04-08 12 -84 val_84 2008-04-08 12 -85 val_85 2008-04-08 11 -85 val_85 2008-04-08 11 -85 val_85 2008-04-08 12 -85 val_85 2008-04-08 12 -86 val_86 2008-04-08 11 -86 val_86 2008-04-08 11 -86 val_86 2008-04-08 12 -86 val_86 2008-04-08 12 -87 val_87 2008-04-08 11 -87 val_87 2008-04-08 11 -87 val_87 2008-04-08 12 -87 val_87 2008-04-08 12 -9 val_9 2008-04-08 11 -9 val_9 2008-04-08 11 -9 val_9 2008-04-08 12 -9 val_9 2008-04-08 12 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 11 -90 val_90 2008-04-08 12 -90 val_90 2008-04-08 12 -90 val_90 2008-04-08 12 -90 val_90 2008-04-08 12 -90 val_90 2008-04-08 12 -90 val_90 2008-04-08 12 -92 val_92 2008-04-08 11 -92 val_92 2008-04-08 11 -92 val_92 2008-04-08 12 -92 val_92 2008-04-08 12 -95 val_95 2008-04-08 11 -95 val_95 2008-04-08 11 -95 val_95 2008-04-08 11 -95 val_95 2008-04-08 11 -95 val_95 2008-04-08 12 -95 val_95 2008-04-08 12 -95 val_95 2008-04-08 12 -95 val_95 2008-04-08 12 -96 val_96 2008-04-08 11 -96 val_96 2008-04-08 11 -96 val_96 2008-04-08 12 -96 val_96 2008-04-08 12 -97 val_97 2008-04-08 11 -97 val_97 2008-04-08 11 -97 val_97 2008-04-08 11 -97 val_97 2008-04-08 11 -97 val_97 2008-04-08 12 -97 val_97 2008-04-08 12 -97 val_97 2008-04-08 12 -97 val_97 2008-04-08 12 -98 val_98 2008-04-08 11 -98 val_98 2008-04-08 11 -98 val_98 2008-04-08 11 -98 val_98 2008-04-08 11 -98 val_98 2008-04-08 12 -98 val_98 2008-04-08 12 -98 val_98 2008-04-08 12 -98 val_98 2008-04-08 12 diff --git ql/src/test/results/clientpositive/union_remove_19.q.out ql/src/test/results/clientpositive/union_remove_19.q.out deleted file mode 100644 index 95819f64d7..0000000000 --- ql/src/test/results/clientpositive/union_remove_19.q.out +++ /dev/null @@ -1,542 +0,0 @@ -PREHOOK: query: create table inputTbl1_n1(key string, val string) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@inputTbl1_n1 -POSTHOOK: query: create table inputTbl1_n1(key string, val string) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@inputTbl1_n1 -PREHOOK: query: create table outputTbl1_n1(key string, `values` bigint) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@outputTbl1_n1 -POSTHOOK: query: create table outputTbl1_n1(key string, `values` bigint) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@outputTbl1_n1 -PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n1 -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@inputtbl1_n1 -POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n1 -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@inputtbl1_n1 -PREHOOK: query: explain -insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: explain -insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - -PREHOOK: query: insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] -PREHOOK: query: desc formatted outputTbl1_n1 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@outputtbl1_n1 -POSTHOOK: query: desc formatted outputTbl1_n1 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@outputtbl1_n1 -# col_name data_type comment -key string -values bigint - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - bucketing_version 2 - numFiles 2 - totalSize 40 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: select * from outputTbl1_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl1_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -1 1 -1 1 -2 1 -2 1 -3 1 -3 1 -7 1 -7 1 -8 2 -8 2 -PREHOOK: query: explain -insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a where a.key = 7 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: explain -insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a where a.key = 7 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - filterExpr: (UDFToDouble(key) = 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - filterExpr: (UDFToDouble(key) = 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) = 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - -PREHOOK: query: insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a where a.key = 7 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: insert overwrite table outputTbl1_n1 -SELECT a.key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a where a.key = 7 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] -PREHOOK: query: select * from outputTbl1_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl1_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -7 1 -7 1 -PREHOOK: query: explain -insert overwrite table outputTbl1_n1 -select key, `values` from -( -SELECT a.key + a.key as key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -) b where b.key >= 7 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: explain -insert overwrite table outputTbl1_n1 -select key, `values` from -( -SELECT a.key + a.key as key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -) b where b.key >= 7 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - filterExpr: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: CAST( (UDFToDouble(_col0) + UDFToDouble(_col0)) AS STRING) (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n1 - filterExpr: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: CAST( (UDFToDouble(_col0) + UDFToDouble(_col0)) AS STRING) (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n1 - -PREHOOK: query: insert overwrite table outputTbl1_n1 -select key, `values` from -( -SELECT a.key + a.key as key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -) b where b.key >= 7 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n1 -PREHOOK: Output: default@outputtbl1_n1 -POSTHOOK: query: insert overwrite table outputTbl1_n1 -select key, `values` from -( -SELECT a.key + a.key as key, a.`values` -FROM ( - SELECT key, count(1) as `values` from inputTbl1_n1 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n1 group by key -) a -) b where b.key >= 7 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n1 -POSTHOOK: Output: default@outputtbl1_n1 -POSTHOOK: Lineage: outputtbl1_n1.key EXPRESSION [(inputtbl1_n1)inputtbl1_n1.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1_n1.values EXPRESSION [(inputtbl1_n1)inputtbl1_n1.null, ] -PREHOOK: query: select * from outputTbl1_n1 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl1_n1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl1_n1 -#### A masked pattern was here #### -14.0 1 -14.0 1 -16.0 2 -16.0 2 diff --git ql/src/test/results/clientpositive/union_remove_23.q.out ql/src/test/results/clientpositive/union_remove_23.q.out deleted file mode 100644 index 0188b56045..0000000000 --- ql/src/test/results/clientpositive/union_remove_23.q.out +++ /dev/null @@ -1,268 +0,0 @@ -PREHOOK: query: create table inputTbl1_n25(key string, val string) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@inputTbl1_n25 -POSTHOOK: query: create table inputTbl1_n25(key string, val string) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@inputTbl1_n25 -PREHOOK: query: create table outputTbl1_n34(key string, `values` bigint) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@outputTbl1_n34 -POSTHOOK: query: create table outputTbl1_n34(key string, `values` bigint) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@outputTbl1_n34 -PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n25 -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@inputtbl1_n25 -POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n25 -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@inputtbl1_n25 -PREHOOK: query: explain -insert overwrite table outputTbl1_n34 -SELECT * -FROM ( - SELECT key, count(1) as `values` from - (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n25 group by key -) subq2 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n25 -PREHOOK: Output: default@outputtbl1_n34 -POSTHOOK: query: explain -insert overwrite table outputTbl1_n34 -SELECT * -FROM ( - SELECT key, count(1) as `values` from - (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n25 group by key -) subq2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n25 -POSTHOOK: Output: default@outputtbl1_n34 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2, Stage-4 - Stage-4 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n34 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n34 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n25 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n34 - -PREHOOK: query: insert overwrite table outputTbl1_n34 -SELECT * -FROM ( - SELECT key, count(1) as `values` from - (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n25 group by key -) subq2 -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n25 -PREHOOK: Output: default@outputtbl1_n34 -POSTHOOK: query: insert overwrite table outputTbl1_n34 -SELECT * -FROM ( - SELECT key, count(1) as `values` from - (SELECT a.key, b.val from inputTbl1_n25 a join inputTbl1_n25 b on a.key=b.key) subq group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n25 group by key -) subq2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n25 -POSTHOOK: Output: default@outputtbl1_n34 -POSTHOOK: Lineage: outputtbl1_n34.key EXPRESSION [(inputtbl1_n25)a.FieldSchema(name:key, type:string, comment:null), (inputtbl1_n25)inputtbl1_n25.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1_n34.values EXPRESSION [(inputtbl1_n25)a.null, (inputtbl1_n25)b.null, (inputtbl1_n25)inputtbl1_n25.null, ] -PREHOOK: query: desc formatted outputTbl1_n34 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@outputtbl1_n34 -POSTHOOK: query: desc formatted outputTbl1_n34 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@outputtbl1_n34 -# col_name data_type comment -key string -values bigint - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - bucketing_version 2 - numFiles 2 - totalSize 40 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: select * from outputTbl1_n34 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl1_n34 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl1_n34 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl1_n34 -#### A masked pattern was here #### -1 1 -1 1 -2 1 -2 1 -3 1 -3 1 -7 1 -7 1 -8 2 -8 4 diff --git ql/src/test/results/clientpositive/union_remove_6_subq.q.out ql/src/test/results/clientpositive/union_remove_6_subq.q.out deleted file mode 100644 index 8b68e4dafe..0000000000 --- ql/src/test/results/clientpositive/union_remove_6_subq.q.out +++ /dev/null @@ -1,1263 +0,0 @@ -PREHOOK: query: create table inputTbl1_n0(key string, val string) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@inputTbl1_n0 -POSTHOOK: query: create table inputTbl1_n0(key string, val string) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@inputTbl1_n0 -PREHOOK: query: create table outputTbl1_n0(key string, `values` bigint) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@outputTbl1_n0 -POSTHOOK: query: create table outputTbl1_n0(key string, `values` bigint) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@outputTbl1_n0 -PREHOOK: query: create table outputTbl2(key string, `values` bigint) stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@outputTbl2 -POSTHOOK: query: create table outputTbl2(key string, `values` bigint) stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@outputTbl2 -PREHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n0 -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@inputtbl1_n0 -POSTHOOK: query: load data local inpath '../../data/files/T1.txt' into table inputTbl1_n0 -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@inputtbl1_n0 -PREHOOK: query: explain -FROM ( - select * from( - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - )subq -) a -insert overwrite table outputTbl1_n0 select * -insert overwrite table outputTbl2 select * -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n0 -PREHOOK: Output: default@outputtbl1_n0 -PREHOOK: Output: default@outputtbl2 -POSTHOOK: query: explain -FROM ( - select * from( - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - )subq -) a -insert overwrite table outputTbl1_n0 select * -insert overwrite table outputTbl2 select * -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n0 -POSTHOOK: Output: default@outputtbl1_n0 -POSTHOOK: Output: default@outputtbl2 -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 - Stage-0 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n0 - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2 - TableScan - Union - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n0 - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1_n0 - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl2 - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: inputtbl1_n0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: FROM ( - select * from( - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - )subq -) a -insert overwrite table outputTbl1_n0 select * -insert overwrite table outputTbl2 select * -PREHOOK: type: QUERY -PREHOOK: Input: default@inputtbl1_n0 -PREHOOK: Output: default@outputtbl1_n0 -PREHOOK: Output: default@outputtbl2 -POSTHOOK: query: FROM ( - select * from( - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - UNION ALL - SELECT key, count(1) as `values` from inputTbl1_n0 group by key - )subq -) a -insert overwrite table outputTbl1_n0 select * -insert overwrite table outputTbl2 select * -POSTHOOK: type: QUERY -POSTHOOK: Input: default@inputtbl1_n0 -POSTHOOK: Output: default@outputtbl1_n0 -POSTHOOK: Output: default@outputtbl2 -POSTHOOK: Lineage: outputtbl1_n0.key EXPRESSION [(inputtbl1_n0)inputtbl1_n0.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1_n0.values EXPRESSION [(inputtbl1_n0)inputtbl1_n0.null, ] -POSTHOOK: Lineage: outputtbl2.key EXPRESSION [(inputtbl1_n0)inputtbl1_n0.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl2.values EXPRESSION [(inputtbl1_n0)inputtbl1_n0.null, ] -PREHOOK: query: select * from outputTbl1_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl1_n0 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl1_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl1_n0 -#### A masked pattern was here #### -1 1 -1 1 -2 1 -2 1 -3 1 -3 1 -7 1 -7 1 -8 2 -8 2 -PREHOOK: query: select * from outputTbl2 -PREHOOK: type: QUERY -PREHOOK: Input: default@outputtbl2 -#### A masked pattern was here #### -POSTHOOK: query: select * from outputTbl2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@outputtbl2 -#### A masked pattern was here #### -1 1 -1 1 -2 1 -2 1 -3 1 -3 1 -7 1 -7 1 -8 2 -8 2 -PREHOOK: query: explain -select avg(c) from( - SELECT count(1)-200 as c from src - UNION ALL - SELECT count(1) as c from src -)subq -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain -select avg(c) from( - SELECT count(1)-200 as c from src - UNION ALL - SELECT count(1) as c from src -)subq -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (_col0 - 200L) (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col0), count(_col0) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col0), count(_col0) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (UDFToDouble(_col0) / _col1) (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select avg(c) from( - SELECT count(1)-200 as c from src - UNION ALL - SELECT count(1) as c from src -)subq -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select avg(c) from( - SELECT count(1)-200 as c from src - UNION ALL - SELECT count(1) as c from src -)subq -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -400.0 -PREHOOK: query: explain -select key, avg(c) over w from( - SELECT key, count(1)*2 as c from src group by key - UNION ALL - SELECT key, count(1) as c from src group by key -)subq group by key, c -WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: explain -select key, avg(c) over w from( - SELECT key, count(1)*2 as c from src group by key - UNION ALL - SELECT key, count(1) as c from src group by key -)subq group by key, c -WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), (_col1 * 2L) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - Union - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: bigint - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: avg_window_0 - arguments: _col1 - name: avg - window function: GenericUDAFAverageEvaluatorDouble - window frame: ROWS PRECEDING(MAX)~CURRENT - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), avg_window_0 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select key, avg(c) over w from( - SELECT key, count(1)*2 as c from src group by key - UNION ALL - SELECT key, count(1) as c from src group by key -)subq group by key, c -WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select key, avg(c) over w from( - SELECT key, count(1)*2 as c from src group by key - UNION ALL - SELECT key, count(1) as c from src group by key -)subq group by key, c -WINDOW w AS (PARTITION BY key ORDER BY c ROWS UNBOUNDED PRECEDING) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 3.0 -0 4.5 -10 1.0 -10 1.5 -100 2.0 -100 3.0 -103 2.0 -103 3.0 -104 2.0 -104 3.0 -105 1.0 -105 1.5 -11 1.0 -11 1.5 -111 1.0 -111 1.5 -113 2.0 -113 3.0 -114 1.0 -114 1.5 -116 1.0 -116 1.5 -118 2.0 -118 3.0 -119 3.0 -119 4.5 -12 2.0 -12 3.0 -120 2.0 -120 3.0 -125 2.0 -125 3.0 -126 1.0 -126 1.5 -128 3.0 -128 4.5 -129 2.0 -129 3.0 -131 1.0 -131 1.5 -133 1.0 -133 1.5 -134 2.0 -134 3.0 -136 1.0 -136 1.5 -137 2.0 -137 3.0 -138 4.0 -138 6.0 -143 1.0 -143 1.5 -145 1.0 -145 1.5 -146 2.0 -146 3.0 -149 2.0 -149 3.0 -15 2.0 -15 3.0 -150 1.0 -150 1.5 -152 2.0 -152 3.0 -153 1.0 -153 1.5 -155 1.0 -155 1.5 -156 1.0 -156 1.5 -157 1.0 -157 1.5 -158 1.0 -158 1.5 -160 1.0 -160 1.5 -162 1.0 -162 1.5 -163 1.0 -163 1.5 -164 2.0 -164 3.0 -165 2.0 -165 3.0 -166 1.0 -166 1.5 -167 3.0 -167 4.5 -168 1.0 -168 1.5 -169 4.0 -169 6.0 -17 1.0 -17 1.5 -170 1.0 -170 1.5 -172 2.0 -172 3.0 -174 2.0 -174 3.0 -175 2.0 -175 3.0 -176 2.0 -176 3.0 -177 1.0 -177 1.5 -178 1.0 -178 1.5 -179 2.0 -179 3.0 -18 2.0 -18 3.0 -180 1.0 -180 1.5 -181 1.0 -181 1.5 -183 1.0 -183 1.5 -186 1.0 -186 1.5 -187 3.0 -187 4.5 -189 1.0 -189 1.5 -19 1.0 -19 1.5 -190 1.0 -190 1.5 -191 2.0 -191 3.0 -192 1.0 -192 1.5 -193 3.0 -193 4.5 -194 1.0 -194 1.5 -195 2.0 -195 3.0 -196 1.0 -196 1.5 -197 2.0 -197 3.0 -199 3.0 -199 4.5 -2 1.0 -2 1.5 -20 1.0 -20 1.5 -200 2.0 -200 3.0 -201 1.0 -201 1.5 -202 1.0 -202 1.5 -203 2.0 -203 3.0 -205 2.0 -205 3.0 -207 2.0 -207 3.0 -208 3.0 -208 4.5 -209 2.0 -209 3.0 -213 2.0 -213 3.0 -214 1.0 -214 1.5 -216 2.0 -216 3.0 -217 2.0 -217 3.0 -218 1.0 -218 1.5 -219 2.0 -219 3.0 -221 2.0 -221 3.0 -222 1.0 -222 1.5 -223 2.0 -223 3.0 -224 2.0 -224 3.0 -226 1.0 -226 1.5 -228 1.0 -228 1.5 -229 2.0 -229 3.0 -230 5.0 -230 7.5 -233 2.0 -233 3.0 -235 1.0 -235 1.5 -237 2.0 -237 3.0 -238 2.0 -238 3.0 -239 2.0 -239 3.0 -24 2.0 -24 3.0 -241 1.0 -241 1.5 -242 2.0 -242 3.0 -244 1.0 -244 1.5 -247 1.0 -247 1.5 -248 1.0 -248 1.5 -249 1.0 -249 1.5 -252 1.0 -252 1.5 -255 2.0 -255 3.0 -256 2.0 -256 3.0 -257 1.0 -257 1.5 -258 1.0 -258 1.5 -26 2.0 -26 3.0 -260 1.0 -260 1.5 -262 1.0 -262 1.5 -263 1.0 -263 1.5 -265 2.0 -265 3.0 -266 1.0 -266 1.5 -27 1.0 -27 1.5 -272 2.0 -272 3.0 -273 3.0 -273 4.5 -274 1.0 -274 1.5 -275 1.0 -275 1.5 -277 4.0 -277 6.0 -278 2.0 -278 3.0 -28 1.0 -28 1.5 -280 2.0 -280 3.0 -281 2.0 -281 3.0 -282 2.0 -282 3.0 -283 1.0 -283 1.5 -284 1.0 -284 1.5 -285 1.0 -285 1.5 -286 1.0 -286 1.5 -287 1.0 -287 1.5 -288 2.0 -288 3.0 -289 1.0 -289 1.5 -291 1.0 -291 1.5 -292 1.0 -292 1.5 -296 1.0 -296 1.5 -298 3.0 -298 4.5 -30 1.0 -30 1.5 -302 1.0 -302 1.5 -305 1.0 -305 1.5 -306 1.0 -306 1.5 -307 2.0 -307 3.0 -308 1.0 -308 1.5 -309 2.0 -309 3.0 -310 1.0 -310 1.5 -311 3.0 -311 4.5 -315 1.0 -315 1.5 -316 3.0 -316 4.5 -317 2.0 -317 3.0 -318 3.0 -318 4.5 -321 2.0 -321 3.0 -322 2.0 -322 3.0 -323 1.0 -323 1.5 -325 2.0 -325 3.0 -327 3.0 -327 4.5 -33 1.0 -33 1.5 -331 2.0 -331 3.0 -332 1.0 -332 1.5 -333 2.0 -333 3.0 -335 1.0 -335 1.5 -336 1.0 -336 1.5 -338 1.0 -338 1.5 -339 1.0 -339 1.5 -34 1.0 -34 1.5 -341 1.0 -341 1.5 -342 2.0 -342 3.0 -344 2.0 -344 3.0 -345 1.0 -345 1.5 -348 5.0 -348 7.5 -35 3.0 -35 4.5 -351 1.0 -351 1.5 -353 2.0 -353 3.0 -356 1.0 -356 1.5 -360 1.0 -360 1.5 -362 1.0 -362 1.5 -364 1.0 -364 1.5 -365 1.0 -365 1.5 -366 1.0 -366 1.5 -367 2.0 -367 3.0 -368 1.0 -368 1.5 -369 3.0 -369 4.5 -37 2.0 -37 3.0 -373 1.0 -373 1.5 -374 1.0 -374 1.5 -375 1.0 -375 1.5 -377 1.0 -377 1.5 -378 1.0 -378 1.5 -379 1.0 -379 1.5 -382 2.0 -382 3.0 -384 3.0 -384 4.5 -386 1.0 -386 1.5 -389 1.0 -389 1.5 -392 1.0 -392 1.5 -393 1.0 -393 1.5 -394 1.0 -394 1.5 -395 2.0 -395 3.0 -396 3.0 -396 4.5 -397 2.0 -397 3.0 -399 2.0 -399 3.0 -4 1.0 -4 1.5 -400 1.0 -400 1.5 -401 5.0 -401 7.5 -402 1.0 -402 1.5 -403 3.0 -403 4.5 -404 2.0 -404 3.0 -406 4.0 -406 6.0 -407 1.0 -407 1.5 -409 3.0 -409 4.5 -41 1.0 -41 1.5 -411 1.0 -411 1.5 -413 2.0 -413 3.0 -414 2.0 -414 3.0 -417 3.0 -417 4.5 -418 1.0 -418 1.5 -419 1.0 -419 1.5 -42 2.0 -42 3.0 -421 1.0 -421 1.5 -424 2.0 -424 3.0 -427 1.0 -427 1.5 -429 2.0 -429 3.0 -43 1.0 -43 1.5 -430 3.0 -430 4.5 -431 3.0 -431 4.5 -432 1.0 -432 1.5 -435 1.0 -435 1.5 -436 1.0 -436 1.5 -437 1.0 -437 1.5 -438 3.0 -438 4.5 -439 2.0 -439 3.0 -44 1.0 -44 1.5 -443 1.0 -443 1.5 -444 1.0 -444 1.5 -446 1.0 -446 1.5 -448 1.0 -448 1.5 -449 1.0 -449 1.5 -452 1.0 -452 1.5 -453 1.0 -453 1.5 -454 3.0 -454 4.5 -455 1.0 -455 1.5 -457 1.0 -457 1.5 -458 2.0 -458 3.0 -459 2.0 -459 3.0 -460 1.0 -460 1.5 -462 2.0 -462 3.0 -463 2.0 -463 3.0 -466 3.0 -466 4.5 -467 1.0 -467 1.5 -468 4.0 -468 6.0 -469 5.0 -469 7.5 -47 1.0 -47 1.5 -470 1.0 -470 1.5 -472 1.0 -472 1.5 -475 1.0 -475 1.5 -477 1.0 -477 1.5 -478 2.0 -478 3.0 -479 1.0 -479 1.5 -480 3.0 -480 4.5 -481 1.0 -481 1.5 -482 1.0 -482 1.5 -483 1.0 -483 1.5 -484 1.0 -484 1.5 -485 1.0 -485 1.5 -487 1.0 -487 1.5 -489 4.0 -489 6.0 -490 1.0 -490 1.5 -491 1.0 -491 1.5 -492 2.0 -492 3.0 -493 1.0 -493 1.5 -494 1.0 -494 1.5 -495 1.0 -495 1.5 -496 1.0 -496 1.5 -497 1.0 -497 1.5 -498 3.0 -498 4.5 -5 3.0 -5 4.5 -51 2.0 -51 3.0 -53 1.0 -53 1.5 -54 1.0 -54 1.5 -57 1.0 -57 1.5 -58 2.0 -58 3.0 -64 1.0 -64 1.5 -65 1.0 -65 1.5 -66 1.0 -66 1.5 -67 2.0 -67 3.0 -69 1.0 -69 1.5 -70 3.0 -70 4.5 -72 2.0 -72 3.0 -74 1.0 -74 1.5 -76 2.0 -76 3.0 -77 1.0 -77 1.5 -78 1.0 -78 1.5 -8 1.0 -8 1.5 -80 1.0 -80 1.5 -82 1.0 -82 1.5 -83 2.0 -83 3.0 -84 2.0 -84 3.0 -85 1.0 -85 1.5 -86 1.0 -86 1.5 -87 1.0 -87 1.5 -9 1.0 -9 1.5 -90 3.0 -90 4.5 -92 1.0 -92 1.5 -95 2.0 -95 3.0 -96 1.0 -96 1.5 -97 2.0 -97 3.0 -98 2.0 -98 3.0 diff --git ql/src/test/results/clientpositive/union_remove_plan.q.out ql/src/test/results/clientpositive/union_remove_plan.q.out deleted file mode 100644 index 373c60f0f5..0000000000 --- ql/src/test/results/clientpositive/union_remove_plan.q.out +++ /dev/null @@ -1,142 +0,0 @@ -PREHOOK: query: create table if not exists test_table(column1 string, column2 int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_table -POSTHOOK: query: create table if not exists test_table(column1 string, column2 int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_table -PREHOOK: query: insert into test_table values('a',1),('b',2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_table -POSTHOOK: query: insert into test_table values('a',1),('b',2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_table -POSTHOOK: Lineage: test_table.column1 SCRIPT [] -POSTHOOK: Lineage: test_table.column2 SCRIPT [] -PREHOOK: query: explain -select column1 from test_table group by column1 -union all -select column1 from test_table group by column1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_table -#### A masked pattern was here #### -POSTHOOK: query: explain -select column1 from test_table group by column1 -union all -select column1 from test_table group by column1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_table -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_table - Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: column1 (type: string) - outputColumnNames: column1 - Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: column1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: test_table - Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: column1 (type: string) - outputColumnNames: column1 - Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: column1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select column1 from test_table group by column1 -union all -select column1 from test_table group by column1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_table -#### A masked pattern was here #### -POSTHOOK: query: select column1 from test_table group by column1 -union all -select column1 from test_table group by column1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_table -#### A masked pattern was here #### -a -b -a -b -PREHOOK: query: drop table test_table -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@test_table -PREHOOK: Output: default@test_table -POSTHOOK: query: drop table test_table -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@test_table -POSTHOOK: Output: default@test_table diff --git ql/src/test/results/clientpositive/union_view.q.out ql/src/test/results/clientpositive/union_view.q.out deleted file mode 100644 index 4c85e7a75c..0000000000 --- ql/src/test/results/clientpositive/union_view.q.out +++ /dev/null @@ -1,1056 +0,0 @@ -PREHOOK: query: CREATE TABLE src_union_1_n0 (key int, value string) PARTITIONED BY (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_union_1_n0 -POSTHOOK: query: CREATE TABLE src_union_1_n0 (key int, value string) PARTITIONED BY (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_union_1_n0 -PREHOOK: query: CREATE TABLE src_union_2_n0 (key int, value string) PARTITIONED BY (ds string, part_1 string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_union_2_n0 -POSTHOOK: query: CREATE TABLE src_union_2_n0 (key int, value string) PARTITIONED BY (ds string, part_1 string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_union_2_n0 -PREHOOK: query: CREATE TABLE src_union_3_n0(key int, value string) PARTITIONED BY (ds string, part_1 string, part_2 string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_union_3_n0 -POSTHOOK: query: CREATE TABLE src_union_3_n0(key int, value string) PARTITIONED BY (ds string, part_1 string, part_2 string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_union_3_n0 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and (ds = '1')) (type: boolean) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), value (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and (ds = '2')) (type: boolean) - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), value (type: string), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and (ds = '3')) (type: boolean) - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), value (type: string), '3' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -86 val_86 1 -86 val_86 2 -86 val_86 2 -86 val_86 3 -86 val_86 3 -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -500 -1000 -1000 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and (ds = '1')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and (ds = '1')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '1')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and (ds = '1')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '1')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and (ds = '2')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '2')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and (ds = '2')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and (ds = '2')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '2')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '2' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and (ds = '3')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '3')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '3' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and (ds = '3')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '3')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '3' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and (ds = '3')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 5 Data size: 441 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '3' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and ds is not null) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 139500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 2 Data size: 558 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 550 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 8 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and ds is not null) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 825 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 8 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and ds is not null) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 825 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 8 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -86 val_86 1 -86 val_86 2 -86 val_86 2 -86 val_86 3 -86 val_86 3 -86 val_86 1 -86 val_86 2 -86 val_86 2 -86 val_86 3 -86 val_86 3 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: (ds = '1') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_2_n0 - filterExpr: (ds = '1') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '1') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_3_n0 - filterExpr: (ds = '1') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '1') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: (ds = '2') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_2_n0 - filterExpr: (ds = '2') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_3_n0 - filterExpr: (ds = '2') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: (ds = '3') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '3') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_2_n0 - filterExpr: (ds = '3') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '3') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_3_n0 - filterExpr: (ds = '3') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1002 Data size: 8016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -500 -1000 -1000 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: ((key = 86) and (ds = '4')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '4')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '4' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_2_n0 - filterExpr: ((key = 86) and (ds = '4')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key = 86) and (ds = '4')) (type: boolean) - Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '4' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: src_union_3_n0 - filterExpr: ((key = 86) and (ds = '4')) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 86) (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 182 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 4 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 86 (type: int), _col0 (type: string), '4' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -86 val_86 4 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_union_1_n0 - filterExpr: (ds = '4') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '4') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_2_n0 - filterExpr: (ds = '4') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '4') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - TableScan - alias: src_union_3_n0 - filterExpr: (ds = '4') (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 502 Data size: 4016 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -500 diff --git ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out deleted file mode 100644 index 663313eab1..0000000000 --- ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out +++ /dev/null @@ -1,138 +0,0 @@ -PREHOOK: query: CREATE temporary TABLE `catalog_Sales`( - `cs_quantity` int, - `cs_wholesale_cost` decimal(7,2), - `cs_list_price` decimal(7,2), - `cs_sales_price` decimal(7,2), - `cs_ext_discount_amt` decimal(7,2), - `cs_ext_sales_price` decimal(7,2), - `cs_ext_wholesale_cost` decimal(7,2), - `cs_ext_list_price` decimal(7,2), - `cs_ext_tax` decimal(7,2), - `cs_coupon_amt` decimal(7,2), - `cs_ext_ship_cost` decimal(7,2), - `cs_net_paid` decimal(7,2), - `cs_net_paid_inc_tax` decimal(7,2), - `cs_net_paid_inc_ship` decimal(7,2), - `cs_net_paid_inc_ship_tax` decimal(7,2), - `cs_net_profit` decimal(7,2)) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@catalog_Sales -POSTHOOK: query: CREATE temporary TABLE `catalog_Sales`( - `cs_quantity` int, - `cs_wholesale_cost` decimal(7,2), - `cs_list_price` decimal(7,2), - `cs_sales_price` decimal(7,2), - `cs_ext_discount_amt` decimal(7,2), - `cs_ext_sales_price` decimal(7,2), - `cs_ext_wholesale_cost` decimal(7,2), - `cs_ext_list_price` decimal(7,2), - `cs_ext_tax` decimal(7,2), - `cs_coupon_amt` decimal(7,2), - `cs_ext_ship_cost` decimal(7,2), - `cs_net_paid` decimal(7,2), - `cs_net_paid_inc_tax` decimal(7,2), - `cs_net_paid_inc_ship` decimal(7,2), - `cs_net_paid_inc_ship_tax` decimal(7,2), - `cs_net_profit` decimal(7,2)) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@catalog_Sales -PREHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales -PREHOOK: type: QUERY -PREHOOK: Input: default@catalog_sales -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales -POSTHOOK: type: QUERY -POSTHOOK: Input: default@catalog_sales -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: catalog_sales - Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cs_quantity:int, 1:cs_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:cs_list_price:decimal(7,2)/DECIMAL_64, 3:cs_sales_price:decimal(7,2)/DECIMAL_64, 4:cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, 5:cs_ext_sales_price:decimal(7,2)/DECIMAL_64, 6:cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 7:cs_ext_list_price:decimal(7,2)/DECIMAL_64, 8:cs_ext_tax:decimal(7,2)/DECIMAL_64, 9:cs_coupon_amt:decimal(7,2)/DECIMAL_64, 10:cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, 11:cs_net_paid:decimal(7,2)/DECIMAL_64, 12:cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, 13:cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, 14:cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, 15:cs_net_profit:decimal(7,2)/DECIMAL_64, 16:ROW__ID:struct] - Select Operator - expressions: ((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) (type: decimal(14,6)) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [20] - selectExpressions: Decimal64ColDivideDecimal64Scalar(col 19:decimal(10,2)/DECIMAL_64, decimal64Val 200, decimalVal 2)(children: Decimal64ColAddDecimal64Column(col 18:decimal(9,2)/DECIMAL_64, col 5:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 17:decimal(8,2)/DECIMAL_64, col 4:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 7:decimal(7,2)/DECIMAL_64, col 6:decimal(7,2)/DECIMAL_64) -> 17:decimal(8,2)/DECIMAL_64) -> 18:decimal(9,2)/DECIMAL_64) -> 19:decimal(10,2)/DECIMAL_64) -> 20:decimal(14,6)/DECIMAL_64 - Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(_col0) - Group By Vectorization: - aggregators: VectorUDAFMaxDecimal64(col 20:decimal(14,6)/DECIMAL_64) -> decimal(14,6)/DECIMAL_64 - className: VectorGroupByOperator - groupByMode: HASH - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(14,6)) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 16 - includeColumns: [4, 5, 6, 7] - dataColumns: cs_quantity:int, cs_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_list_price:decimal(7,2)/DECIMAL_64, cs_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, cs_ext_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_ext_list_price:decimal(7,2)/DECIMAL_64, cs_ext_tax:decimal(7,2)/DECIMAL_64, cs_coupon_amt:decimal(7,2)/DECIMAL_64, cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, cs_net_paid:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, cs_net_profit:decimal(7,2)/DECIMAL_64 - partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, decimal(14,6)/DECIMAL_64] - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git ql/src/test/results/clientpositive/vector_decimal_partition.q.out ql/src/test/results/clientpositive/vector_decimal_partition.q.out deleted file mode 100644 index bc85edfb7f..0000000000 --- ql/src/test/results/clientpositive/vector_decimal_partition.q.out +++ /dev/null @@ -1,153 +0,0 @@ -PREHOOK: query: DROP TABLE IF EXISTS decimal_part -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS decimal_part -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE decimal_part (id DECIMAL(4,0), foo VARCHAR(10)) - PARTITIONED BY (nr_bank DECIMAL(4,0)) -STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@decimal_part -POSTHOOK: query: CREATE TABLE decimal_part (id DECIMAL(4,0), foo VARCHAR(10)) - PARTITIONED BY (nr_bank DECIMAL(4,0)) -STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@decimal_part -PREHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 88) VALUES (1, 'test') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@decimal_part@nr_bank=88 -POSTHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 88) VALUES (1, 'test') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@decimal_part@nr_bank=88 -POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=88).foo SCRIPT [] -POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=88).id SCRIPT [] -PREHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 8801) VALUES (1, '8801') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@decimal_part@nr_bank=8801 -POSTHOOK: query: INSERT INTO decimal_part PARTITION (nr_bank = 8801) VALUES (1, '8801') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@decimal_part@nr_bank=8801 -POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=8801).foo SCRIPT [] -POSTHOOK: Lineage: decimal_part PARTITION(nr_bank=8801).id SCRIPT [] -PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_part -PREHOOK: Input: default@decimal_part@nr_bank=88 -PREHOOK: Input: default@decimal_part@nr_bank=8801 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_part -POSTHOOK: Input: default@decimal_part@nr_bank=88 -POSTHOOK: Input: default@decimal_part@nr_bank=8801 -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: decimal_part - Statistics: Num rows: 2 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: nr_bank (type: decimal(4,0)) - outputColumnNames: nr_bank - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 2 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - Group By Vectorization: - aggregators: VectorUDAFCountStar(*) -> bigint - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: ConvertDecimal64ToDecimal(col 2:decimal(4,0)/DECIMAL_64) -> 4:decimal(4,0) - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - keys: nr_bank (type: decimal(4,0)) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: decimal(4,0)) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: decimal(4,0)) - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: decimal(4,0)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: bigint), _col0 (type: decimal(4,0)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_part -PREHOOK: Input: default@decimal_part@nr_bank=88 -PREHOOK: Input: default@decimal_part@nr_bank=8801 -#### A masked pattern was here #### -POSTHOOK: query: SELECT count(*), nr_bank FROM decimal_part GROUP BY nr_bank -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_part -POSTHOOK: Input: default@decimal_part@nr_bank=88 -POSTHOOK: Input: default@decimal_part@nr_bank=8801 -#### A masked pattern was here #### -1 88 -1 8801 diff --git ql/src/test/results/clientpositive/vector_delete_orig_table.q.out ql/src/test/results/clientpositive/vector_delete_orig_table.q.out deleted file mode 100644 index c2d96af845..0000000000 --- ql/src/test/results/clientpositive/vector_delete_orig_table.q.out +++ /dev/null @@ -1,167 +0,0 @@ -PREHOOK: query: create table acid_dot( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - ctimestamp2 TIMESTAMP, - cboolean1 BOOLEAN, -#### A masked pattern was here #### -PREHOOK: type: CREATETABLE -#### A masked pattern was here #### -PREHOOK: Output: database:default -PREHOOK: Output: default@acid_dot -POSTHOOK: query: create table acid_dot( - ctinyint TINYINT, - csmallint SMALLINT, - cint INT, - cbigint BIGINT, - cfloat FLOAT, - cdouble DOUBLE, - cstring1 STRING, - cstring2 STRING, - ctimestamp1 TIMESTAMP, - ctimestamp2 TIMESTAMP, - cboolean1 BOOLEAN, -#### A masked pattern was here #### -POSTHOOK: type: CREATETABLE -#### A masked pattern was here #### -POSTHOOK: Output: database:default -POSTHOOK: Output: default@acid_dot -PREHOOK: query: explain vectorization detail -select count(*) from acid_dot -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_dot -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization detail -select count(*) from acid_dot -POSTHOOK: type: QUERY -POSTHOOK: Input: default@acid_dot -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: acid_dot - Statistics: Num rows: 5865 Data size: 2956160 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [] - Statistics: Num rows: 5865 Data size: 2956160 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - Group By Vectorization: - aggregators: VectorUDAFCountStar(*) -> bigint - className: VectorGroupByOperator - groupByMode: HASH - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 12 - includeColumns: [] - dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from acid_dot -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_dot -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from acid_dot -POSTHOOK: type: QUERY -POSTHOOK: Input: default@acid_dot -#### A masked pattern was here #### -12288 -PREHOOK: query: select count(*) from acid_dot where cint < -1070551679 -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_dot -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from acid_dot where cint < -1070551679 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@acid_dot -#### A masked pattern was here #### -8 -PREHOOK: query: delete from acid_dot where cint < -1070551679 -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_dot -PREHOOK: Output: default@acid_dot -POSTHOOK: query: delete from acid_dot where cint < -1070551679 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@acid_dot -POSTHOOK: Output: default@acid_dot -PREHOOK: query: select count(*) from acid_dot -PREHOOK: type: QUERY -PREHOOK: Input: default@acid_dot -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from acid_dot -POSTHOOK: type: QUERY -POSTHOOK: Input: default@acid_dot -#### A masked pattern was here #### -12280 -#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/vector_empty_where.q.out ql/src/test/results/clientpositive/vector_empty_where.q.out deleted file mode 100644 index ebb75bb277..0000000000 --- ql/src/test/results/clientpositive/vector_empty_where.q.out +++ /dev/null @@ -1,624 +0,0 @@ -PREHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cstring1 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cstring1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 899146 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastStringToBoolean(col 6) -> 13:boolean) - predicate: cstring1 (type: string) - Statistics: Num rows: 6144 Data size: 449620 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: cint - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 6144 Data size: 449620 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: col 2:int - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [] - keys: cint (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col0) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - TableScan Vectorization: - native: true - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count (distinct cint) from alltypesorc where cstring1 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select count (distinct cint) from alltypesorc where cstring1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -6041 -PREHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cint -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cint -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastLongToBooleanViaLongToLong(col 2:int) -> 13:boolean) - predicate: cint (type: int) - Statistics: Num rows: 6144 Data size: 18348 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: col 2:int - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [] - keys: cint (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 3053 Data size: 9120 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col0) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - TableScan Vectorization: - native: true - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count (distinct cint) from alltypesorc where cint -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select count (distinct cint) from alltypesorc where cint -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -6082 -PREHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 73392 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastDoubleToBooleanViaDoubleToLong(col 4:float) -> 13:boolean) - predicate: cfloat (type: float) - Statistics: Num rows: 6144 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: cint - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 6144 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: col 2:int - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [] - keys: cint (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col0) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - TableScan Vectorization: - native: true - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count (distinct cint) from alltypesorc where cfloat -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select count (distinct cint) from alltypesorc where cfloat -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -3022 -PREHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where ctimestamp1 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select count (distinct cint) from alltypesorc where ctimestamp1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 403656 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: CastTimestampToBoolean(col 8:timestamp) -> 13:boolean) - predicate: ctimestamp1 (type: timestamp) - Statistics: Num rows: 6144 Data size: 201828 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: cint - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 6144 Data size: 201828 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: col 2:int - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [] - keys: cint (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 3072 Data size: 9176 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col0) - mode: partial2 - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - TableScan Vectorization: - native: true - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count (distinct cint) from alltypesorc where ctimestamp1 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select count (distinct cint) from alltypesorc where ctimestamp1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -3022 diff --git ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out deleted file mode 100644 index 58fa5b3b70..0000000000 --- ql/src/test/results/clientpositive/vector_non_constant_in_expr.q.out +++ /dev/null @@ -1,55 +0,0 @@ -PREHOOK: query: explain vectorization SELECT * FROM alltypesorc WHERE cint in (ctinyint, cbigint) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization SELECT * FROM alltypesorc WHERE cint in (ctinyint, cbigint) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ((cint = UDFToInteger(ctinyint)) or (UDFToLong(cint) = cbigint)) (type: boolean) - Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((cint = UDFToInteger(ctinyint)) or (UDFToLong(cint) = cbigint)) (type: boolean) - Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 12288 Data size: 2844090 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git ql/src/test/results/clientpositive/vector_outer_join_no_keys.q.out ql/src/test/results/clientpositive/vector_outer_join_no_keys.q.out deleted file mode 100644 index 3aaeee4138..0000000000 --- ql/src/test/results/clientpositive/vector_outer_join_no_keys.q.out +++ /dev/null @@ -1,333 +0,0 @@ -PREHOOK: query: create temporary table foo(x int) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@foo -POSTHOOK: query: create temporary table foo(x int) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@foo -PREHOOK: query: insert into foo values(1),(2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@foo -POSTHOOK: query: insert into foo values(1),(2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@foo -POSTHOOK: Lineage: foo.x SCRIPT [] -PREHOOK: query: create temporary table bar(y int) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@bar -POSTHOOK: query: create temporary table bar(y int) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@bar -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-2:MAPRED' is a cross product -PREHOOK: query: explain vectorization detail -select count(*) from bar right outer join foo -PREHOOK: type: QUERY -PREHOOK: Input: default@bar -PREHOOK: Input: default@foo -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization detail -select count(*) from bar right outer join foo -POSTHOOK: type: QUERY -POSTHOOK: Input: default@bar -POSTHOOK: Input: default@foo -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:bar - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:bar - TableScan - alias: bar - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 - 1 {true} - keys: - 0 - 1 - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: foo - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:x:int, 1:ROW__ID:struct] - Select Operator - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [] - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - filter predicates: - 0 - 1 {true} - keys: - 0 - 1 - Map Join Vectorization: - className: VectorMapJoinOuterFilteredOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, Outer Join has keys IS false - Statistics: Num rows: 2 Data size: 10 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - Group By Vectorization: - aggregators: VectorUDAFCountStar(*) -> bigint - className: VectorGroupByOperator - groupByMode: HASH - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: x:int - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Local Work: - Map Reduce Local Work - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-2:MAPRED' is a cross product -PREHOOK: query: -- = 2 - -select count(*) from bar right outer join foo -PREHOOK: type: QUERY -PREHOOK: Input: default@bar -PREHOOK: Input: default@foo -#### A masked pattern was here #### -POSTHOOK: query: -- = 2 - -select count(*) from bar right outer join foo -POSTHOOK: type: QUERY -POSTHOOK: Input: default@bar -POSTHOOK: Input: default@foo -#### A masked pattern was here #### -2 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-2:MAPRED' is a cross product -PREHOOK: query: -- = 2 - -explain vectorization detail -select count(*) from bar, foo -PREHOOK: type: QUERY -PREHOOK: Input: default@bar -PREHOOK: Input: default@foo -#### A masked pattern was here #### -POSTHOOK: query: -- = 2 - -explain vectorization detail -select count(*) from bar, foo -POSTHOOK: type: QUERY -POSTHOOK: Input: default@bar -POSTHOOK: Input: default@foo -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:bar - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:bar - TableScan - alias: bar - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: foo - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:x:int, 1:ROW__ID:struct] - Select Operator - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [] - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - Map Join Vectorization: - className: VectorMapJoinOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 2 Data size: 10 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count() - Group By Vectorization: - aggregators: VectorUDAFCountStar(*) -> bigint - className: VectorGroupByOperator - groupByMode: HASH - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: x:int - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Local Work: - Map Reduce Local Work - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-2:MAPRED' is a cross product -PREHOOK: query: -- = 0 - -select count(*) from bar, foo -PREHOOK: type: QUERY -PREHOOK: Input: default@bar -PREHOOK: Input: default@foo -#### A masked pattern was here #### -POSTHOOK: query: -- = 0 - -select count(*) from bar, foo -POSTHOOK: type: QUERY -POSTHOOK: Input: default@bar -POSTHOOK: Input: default@foo -#### A masked pattern was here #### -0 diff --git ql/src/test/results/clientpositive/vector_tablesample_rows.q.out ql/src/test/results/clientpositive/vector_tablesample_rows.q.out deleted file mode 100644 index 953e0934c5..0000000000 --- ql/src/test/results/clientpositive/vector_tablesample_rows.q.out +++ /dev/null @@ -1,451 +0,0 @@ -PREHOOK: query: explain vectorization detail -select 'key1', 'value1' from alltypesorc tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization detail -select 'key1', 'value1' from alltypesorc tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Row Limit Per Split: 1 - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: 'key1' (type: string), 'value1' (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [13, 14] - selectExpressions: ConstantVectorExpression(val key1) -> 13:string, ConstantVectorExpression(val value1) -> 14:string - Statistics: Num rows: 12288 Data size: 2187264 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 12288 Data size: 2187264 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 12 - includeColumns: [] - dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean - partitionColumnCount: 0 - scratchColumnTypeNames: [string, string] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select 'key1', 'value1' from alltypesorc tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select 'key1', 'value1' from alltypesorc tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -_c0 _c1 -key1 value1 -PREHOOK: query: create table decimal_2_n0 (t decimal(18,9)) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@decimal_2_n0 -POSTHOOK: query: create table decimal_2_n0 (t decimal(18,9)) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@decimal_2_n0 -PREHOOK: query: explain vectorization detail -insert overwrite table decimal_2_n0 - select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@decimal_2_n0 -POSTHOOK: query: explain vectorization detail -insert overwrite table decimal_2_n0 - select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@decimal_2_n0 -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Row Limit Per Split: 1 - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: 17.29 (type: decimal(18,9)) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [13] - selectExpressions: ConstantVectorExpression(val 17.29) -> 13:decimal(18,9) - Statistics: Num rows: 12288 Data size: 1376256 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 12288 Data size: 1376256 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.decimal_2_n0 - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 12 - includeColumns: [] - dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean - partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(18,9)] - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.decimal_2_n0 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: insert overwrite table decimal_2_n0 - select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Output: default@decimal_2_n0 -POSTHOOK: query: insert overwrite table decimal_2_n0 - select cast('17.29' as decimal(4,2)) from alltypesorc tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Output: default@decimal_2_n0 -POSTHOOK: Lineage: decimal_2_n0.t EXPRESSION [] -_col0 -PREHOOK: query: select count(*) from decimal_2_n0 -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_2_n0 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from decimal_2_n0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_2_n0 -#### A masked pattern was here #### -_c0 -1 -PREHOOK: query: drop table decimal_2_n0 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@decimal_2_n0 -PREHOOK: Output: default@decimal_2_n0 -POSTHOOK: query: drop table decimal_2_n0 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@decimal_2_n0 -POSTHOOK: Output: default@decimal_2_n0 -PREHOOK: query: explain vectorization detail -select count(1) from (select * from (Select 1 a) x order by x.a) y -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization detail -select count(1) from (select * from (Select 1 a) x order by x.a) y -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Map Vectorization: - enabled: false - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - enabledConditionsNotMet: Could not enable vectorization due to partition column names size 1 is greater than the number of table column names size 0 IS false - inputFileFormats: org.apache.hadoop.hive.ql.io.NullRowsInputFormat - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(1) from (select * from (Select 1 a) x order by x.a) y -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select count(1) from (select * from (Select 1 a) x order by x.a) y -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -_c0 -1 -PREHOOK: query: explain vectorization detail -create temporary table dual as select 1 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@dual -POSTHOOK: query: explain vectorization detail -create temporary table dual as select 1 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dual -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-8 depends on stages: Stage-0 - Stage-2 depends on stages: Stage-8 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dual - Map Vectorization: - enabled: false - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - enabledConditionsNotMet: Could not enable vectorization due to partition column names size 1 is greater than the number of table column names size 0 IS false - inputFileFormats: org.apache.hadoop.hive.ql.io.NullRowsInputFormat - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-8 - Create Table - columns: _c0 int - name: default.dual - input format: org.apache.hadoop.mapred.TextInputFormat -#### A masked pattern was here #### - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - isTemporary: true - - Stage: Stage-2 - Stats Work - Basic Stats Work: - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dual - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dual - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: create temporary table dual as select 1 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@dual -POSTHOOK: query: create temporary table dual as select 1 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dual -_c0 -PREHOOK: query: select * from dual -PREHOOK: type: QUERY -PREHOOK: Input: default@dual -#### A masked pattern was here #### -POSTHOOK: query: select * from dual -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dual -#### A masked pattern was here #### -dual._c0 -1 diff --git ql/src/test/results/clientpositive/vector_windowing_row_number.q.out ql/src/test/results/clientpositive/vector_windowing_row_number.q.out deleted file mode 100644 index 81ffa0229f..0000000000 --- ql/src/test/results/clientpositive/vector_windowing_row_number.q.out +++ /dev/null @@ -1,926 +0,0 @@ -PREHOOK: query: drop table row_number_test -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table row_number_test -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table row_number_test as select explode(split(repeat("w,", 2400), ",")) -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: database:default -PREHOOK: Output: default@row_number_test -POSTHOOK: query: create table row_number_test as select explode(split(repeat("w,", 2400), ",")) -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: database:default -POSTHOOK: Output: default@row_number_test -POSTHOOK: Lineage: row_number_test.col SCRIPT [] -col -PREHOOK: query: insert into row_number_test select explode(split(repeat("x,", 1200), ",")) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@row_number_test -POSTHOOK: query: insert into row_number_test select explode(split(repeat("x,", 1200), ",")) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@row_number_test -POSTHOOK: Lineage: row_number_test.col SCRIPT [] -col -PREHOOK: query: insert into row_number_test select explode(split(repeat("y,", 700), ",")) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@row_number_test -POSTHOOK: query: insert into row_number_test select explode(split(repeat("y,", 700), ",")) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@row_number_test -POSTHOOK: Lineage: row_number_test.col SCRIPT [] -col -PREHOOK: query: insert into row_number_test select explode(split(repeat("z,", 600), ",")) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@row_number_test -POSTHOOK: query: insert into row_number_test select explode(split(repeat("z,", 600), ",")) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@row_number_test -POSTHOOK: Lineage: row_number_test.col SCRIPT [] -col -PREHOOK: query: explain select - row_number() over() as r1, - row_number() over(order by col) r2, - row_number() over(partition by col) r3, - row_number() over(partition by col order by col) r4, - row_number() over(partition by 1 order by col) r5, - row_number() over(partition by col order by 2) r6, - row_number() over(partition by 1 order by 2) r7, - col - from row_number_test -PREHOOK: type: QUERY -PREHOOK: Input: default@row_number_test -#### A masked pattern was here #### -POSTHOOK: query: explain select - row_number() over() as r1, - row_number() over(order by col) r2, - row_number() over(partition by col) r3, - row_number() over(partition by col order by col) r4, - row_number() over(partition by 1 order by col) r5, - row_number() over(partition by col order by 2) r6, - row_number() over(partition by 1 order by 2) r7, - col - from row_number_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@row_number_test -#### A masked pattern was here #### -Explain -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-4 - Stage-6 depends on stages: Stage-5 - Stage-7 depends on stages: Stage-6 - Stage-0 depends on stages: Stage-7 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: row_number_test - Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: 0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: col (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 0 ASC NULLS FIRST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: row_number_window_0 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_0 (type: int), _col0 (type: string) - outputColumnNames: row_number_window_0, _col0 - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 0 (type: int), _col0 (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_0 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: row_number_window_1 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_1 (type: int), _col0 (type: int), _col1 (type: string) - outputColumnNames: row_number_window_1, _col0, _col1 - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_1 (type: int), _col0 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 ASC NULLS FIRST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: row_number_window_2 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: string) - outputColumnNames: row_number_window_2, _col0, _col1, _col2 - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col3 ASC NULLS LAST - partition by: _col3 - raw input shape: - window functions: - window function definition - alias: row_number_window_3 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) - outputColumnNames: row_number_window_3, _col0, _col1, _col2, _col3 - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 1 (type: int), _col3 (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col4 ASC NULLS LAST - partition by: 1 - raw input shape: - window functions: - window function definition - alias: row_number_window_4 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: string) - outputColumnNames: row_number_window_4, _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col4 (type: string), 2 (type: int) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 2 ASC NULLS LAST - partition by: _col5 - raw input shape: - window functions: - window function definition - alias: row_number_window_5 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: row_number_window_5, _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 1 (type: int), 2 (type: int) - null sort order: az - sort order: ++ - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: int, _col6: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 2 ASC NULLS LAST - partition by: 1 - raw input shape: - window functions: - window function definition - alias: row_number_window_6 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: int), _col4 (type: int), _col3 (type: int), _col2 (type: int), _col1 (type: int), _col0 (type: int), row_number_window_6 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: create table row_numbers_vectorized as select -row_number() over() as r1, -row_number() over(order by col) r2, -row_number() over(partition by col) r3, -row_number() over(partition by col order by col) r4, -row_number() over(partition by 1 order by col) r5, -row_number() over(partition by col order by 2) r6, -row_number() over(partition by 1 order by 2) r7, -col -from row_number_test -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@row_number_test -PREHOOK: Output: database:default -PREHOOK: Output: default@row_numbers_vectorized -POSTHOOK: query: create table row_numbers_vectorized as select -row_number() over() as r1, -row_number() over(order by col) r2, -row_number() over(partition by col) r3, -row_number() over(partition by col order by col) r4, -row_number() over(partition by 1 order by col) r5, -row_number() over(partition by col order by 2) r6, -row_number() over(partition by 1 order by 2) r7, -col -from row_number_test -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@row_number_test -POSTHOOK: Output: database:default -POSTHOOK: Output: default@row_numbers_vectorized -POSTHOOK: Lineage: row_numbers_vectorized.col SIMPLE [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r1 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r2 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r3 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r4 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r5 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r6 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_vectorized.r7 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -r1 r2 r3 r4 r5 r6 r7 col -PREHOOK: query: explain select - row_number() over() as r1, - row_number() over(order by col) r2, - row_number() over(partition by col) r3, - row_number() over(partition by col order by col) r4, - row_number() over(partition by 1 order by col) r5, - row_number() over(partition by col order by 2) r6, - row_number() over(partition by 1 order by 2) r7, - col - from row_number_test -PREHOOK: type: QUERY -PREHOOK: Input: default@row_number_test -#### A masked pattern was here #### -POSTHOOK: query: explain select - row_number() over() as r1, - row_number() over(order by col) r2, - row_number() over(partition by col) r3, - row_number() over(partition by col order by col) r4, - row_number() over(partition by 1 order by col) r5, - row_number() over(partition by col order by 2) r6, - row_number() over(partition by 1 order by 2) r7, - col - from row_number_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@row_number_test -#### A masked pattern was here #### -Explain -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-5 depends on stages: Stage-4 - Stage-6 depends on stages: Stage-5 - Stage-7 depends on stages: Stage-6 - Stage-0 depends on stages: Stage-7 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: row_number_test - Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: 0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 4904 Data size: 416840 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: col (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 0 ASC NULLS FIRST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: row_number_window_0 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_0 (type: int), _col0 (type: string) - outputColumnNames: row_number_window_0, _col0 - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 0 (type: int), _col0 (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 4904 Data size: 1731112 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_0 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: row_number_window_1 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_1 (type: int), _col0 (type: int), _col1 (type: string) - outputColumnNames: row_number_window_1, _col0, _col1 - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 4904 Data size: 1750728 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_1 (type: int), _col0 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 ASC NULLS FIRST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: row_number_window_2 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: string) - outputColumnNames: row_number_window_2, _col0, _col1, _col2 - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 4904 Data size: 1770344 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_2 (type: int), _col0 (type: int), _col1 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col3 ASC NULLS LAST - partition by: _col3 - raw input shape: - window functions: - window function definition - alias: row_number_window_3 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) - outputColumnNames: row_number_window_3, _col0, _col1, _col2, _col3 - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 1 (type: int), _col3 (type: string) - null sort order: az - sort order: ++ - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 4904 Data size: 1789960 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_3 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col4 ASC NULLS LAST - partition by: 1 - raw input shape: - window functions: - window function definition - alias: row_number_window_4 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: string) - outputColumnNames: row_number_window_4, _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col4 (type: string), 2 (type: int) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 4904 Data size: 1809576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_4 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 2 ASC NULLS LAST - partition by: _col5 - raw input shape: - window functions: - window function definition - alias: row_number_window_5 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: row_number_window_5, _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 1 (type: int), 2 (type: int) - null sort order: az - sort order: ++ - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 4904 Data size: 1829192 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: row_number_window_5 (type: int), _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col2: int, _col3: int, _col4: int, _col5: int, _col6: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: 2 ASC NULLS LAST - partition by: 1 - raw input shape: - window functions: - window function definition - alias: row_number_window_6 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 4904 Data size: 1848808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: int), _col4 (type: int), _col3 (type: int), _col2 (type: int), _col1 (type: int), _col0 (type: int), row_number_window_6 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4904 Data size: 554152 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: create table row_numbers_non_vectorized as select -row_number() over() as r1, -row_number() over(order by col) r2, -row_number() over(partition by col) r3, -row_number() over(partition by col order by col) r4, -row_number() over(partition by 1 order by col) r5, -row_number() over(partition by col order by 2) r6, -row_number() over(partition by 1 order by 2) r7, -col -from row_number_test -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@row_number_test -PREHOOK: Output: database:default -PREHOOK: Output: default@row_numbers_non_vectorized -POSTHOOK: query: create table row_numbers_non_vectorized as select -row_number() over() as r1, -row_number() over(order by col) r2, -row_number() over(partition by col) r3, -row_number() over(partition by col order by col) r4, -row_number() over(partition by 1 order by col) r5, -row_number() over(partition by col order by 2) r6, -row_number() over(partition by 1 order by 2) r7, -col -from row_number_test -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@row_number_test -POSTHOOK: Output: database:default -POSTHOOK: Output: default@row_numbers_non_vectorized -POSTHOOK: Lineage: row_numbers_non_vectorized.col SIMPLE [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r1 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r2 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r3 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r4 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r5 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r6 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -POSTHOOK: Lineage: row_numbers_non_vectorized.r7 SCRIPT [(row_number_test)row_number_test.FieldSchema(name:col, type:string, comment:null), ] -r1 r2 r3 r4 r5 r6 r7 col -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: select exists( -select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_vectorized -minus -select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_non_vectorized -) diff_exists -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Input: default@row_numbers_non_vectorized -PREHOOK: Input: default@row_numbers_vectorized -#### A masked pattern was here #### -POSTHOOK: query: select exists( -select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_vectorized -minus -select r1, r2, r3, r4, r5, r6, r7, col from row_numbers_non_vectorized -) diff_exists -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Input: default@row_numbers_non_vectorized -POSTHOOK: Input: default@row_numbers_vectorized -#### A masked pattern was here #### -diff_exists -false -PREHOOK: query: drop table row_numbers_non_vectorized -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@row_numbers_non_vectorized -PREHOOK: Output: default@row_numbers_non_vectorized -POSTHOOK: query: drop table row_numbers_non_vectorized -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@row_numbers_non_vectorized -POSTHOOK: Output: default@row_numbers_non_vectorized -PREHOOK: query: drop table row_numbers_vectorized -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@row_numbers_vectorized -PREHOOK: Output: default@row_numbers_vectorized -POSTHOOK: query: drop table row_numbers_vectorized -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@row_numbers_vectorized -POSTHOOK: Output: default@row_numbers_vectorized -PREHOOK: query: drop table row_number_test -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@row_number_test -PREHOOK: Output: default@row_number_test -POSTHOOK: query: drop table row_number_test -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@row_number_test -POSTHOOK: Output: default@row_number_test diff --git ql/src/test/results/clientpositive/vectorization_multi_value.q.out ql/src/test/results/clientpositive/vectorization_multi_value.q.out deleted file mode 100644 index b4507fb101..0000000000 --- ql/src/test/results/clientpositive/vectorization_multi_value.q.out +++ /dev/null @@ -1,653 +0,0 @@ -PREHOOK: query: DROP TABLE IF EXISTS cond_vector -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS cond_vector -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE cond_vector(a STRING) STORED AS ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cond_vector -POSTHOOK: query: CREATE TABLE cond_vector(a STRING) STORED AS ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cond_vector -PREHOOK: query: INSERT OVERWRITE TABLE cond_vector VALUES("a/b"),("a/b"),("c/d") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cond_vector -POSTHOOK: query: INSERT OVERWRITE TABLE cond_vector VALUES("a/b"),("a/b"),("c/d") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cond_vector -POSTHOOK: Lineage: cond_vector.a SCRIPT [] -PREHOOK: query: SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -{"a":"b"} -{"a":"b"} -{"a":"b"} -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", "b"), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, map('a':'b'), null) (type: map) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [4] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:map, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':'b')) -> 3:map) -> 4:map - Statistics: Num rows: 3 Data size: 326 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 326 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, map, map] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -{"a":{"b":"c"}} -{"a":{"b":"c"}} -{"a":{"b":"c"}} -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b","c")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, map('a':map('b':'c')), null) (type: map>) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 4:map>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':map('b':'c')))(children: VectorUDFAdaptor(map('b':'c')) -> 3:map) -> 4:map>) -> 5:map> - Statistics: Num rows: 3 Data size: 241 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 241 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, map, map>, map>] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -{"a":"a/b"} -{"a":"a/b"} -{"a":"c/d"} -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", a), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, map('a':a), null) (type: map) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [4] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:map, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':a)) -> 3:map) -> 4:map - Statistics: Num rows: 3 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 920 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [0] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, map, map] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -{"a":{"b":"a/b"}} -{"a":{"b":"a/b"}} -{"a":{"b":"c/d"}} -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, MAP("a", MAP("b", a)), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, map('a':map('b':a)), null) (type: map>) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 4:map>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(map('a':map('b':a)))(children: VectorUDFAdaptor(map('b':a)) -> 3:map) -> 4:map>) -> 5:map> - Statistics: Num rows: 3 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 1656 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [0] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, map, map>, map>] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -["a","b"] -["a","b"] -["a","b"] -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", "b"), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, array('a','b'), null) (type: array) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [4] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:array, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array('a','b')) -> 3:array) -> 4:array - Statistics: Num rows: 3 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, array, array] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -[["a","b"],["c","d"]] -[["a","b"],["c","d"]] -[["a","b"],["c","d"]] -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", "b"), ARRAY("c", "d")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, array(array('a','b'),array('c','d')), null) (type: array>) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [6] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 5:array>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array(array('a','b'),array('c','d')))(children: VectorUDFAdaptor(array('a','b')) -> 3:array, VectorUDFAdaptor(array('c','d')) -> 4:array) -> 5:array>) -> 6:array> - Statistics: Num rows: 3 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, array, array, array>, array>] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -["a","a/b"] -["a","a/b"] -["a","c/d"] -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY("a", a), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, array('a',a), null) (type: array) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [4] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 3:array, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array('a',a)) -> 3:array) -> 4:array - Statistics: Num rows: 3 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [0] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, array, array] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -[["a","a/b"],["b","c"]] -[["a","a/b"],["b","c"]] -[["a","c/d"],["b","c"]] -PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector -PREHOOK: type: QUERY -PREHOOK: Input: default@cond_vector -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT IF(1=1, ARRAY(ARRAY("a", a), ARRAY("b", "c")), NULL) FROM cond_vector -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cond_vector -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cond_vector - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:a:string, 1:ROW__ID:struct] - Select Operator - expressions: if(true, array(array('a',a),array('b','c')), null) (type: array>) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [6] - selectExpressions: IfExprCondExprNull(col 2:boolean, col 5:array>, null)(children: ConstantVectorExpression(val 1) -> 2:boolean, VectorUDFAdaptor(array(array('a',a),array('b','c')))(children: VectorUDFAdaptor(array('a',a)) -> 3:array, VectorUDFAdaptor(array('b','c')) -> 4:array) -> 5:array>) -> 6:array> - Statistics: Num rows: 3 Data size: 19280 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 3 Data size: 19280 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: true - vectorized: true - rowBatchContext: - dataColumnCount: 1 - includeColumns: [0] - dataColumns: a:string - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, array, array, array>, array>] - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git ql/src/test/results/clientpositive/vectorization_numeric_overflows.q.out ql/src/test/results/clientpositive/vectorization_numeric_overflows.q.out deleted file mode 100644 index f369d7544f..0000000000 --- ql/src/test/results/clientpositive/vectorization_numeric_overflows.q.out +++ /dev/null @@ -1,1210 +0,0 @@ -PREHOOK: query: CREATE TABLE test_overflow ( - ctinyint1 TINYINT, - ctinyint2 TINYINT, - csmallint1 SMALLINT, - csmallint2 SMALLINT, - cint1 INT, - cint2 INT, - cbigint1 BIGINT, - cbigint2 BIGINT, - cfloat1 FLOAT, - cfloat2 FLOAT, - cdouble1 DOUBLE, - cdouble2 DOUBLE) -STORED AS PARQUET -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_overflow -POSTHOOK: query: CREATE TABLE test_overflow ( - ctinyint1 TINYINT, - ctinyint2 TINYINT, - csmallint1 SMALLINT, - csmallint2 SMALLINT, - cint1 INT, - cint2 INT, - cbigint1 BIGINT, - cbigint2 BIGINT, - cfloat1 FLOAT, - cfloat2 FLOAT, - cdouble1 DOUBLE, - cdouble2 DOUBLE) -STORED AS PARQUET -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_overflow -PREHOOK: query: insert into test_overflow values (-128, 127, -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, 1.401298464324817E-45, 3.4028234663852886E38, 4.9E-324, 1.7976931348623157E308) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_overflow -POSTHOOK: query: insert into test_overflow values (-128, 127, -32768, 32767, -2147483648, 2147483647, -9223372036854775808, 9223372036854775807, 1.401298464324817E-45, 3.4028234663852886E38, 4.9E-324, 1.7976931348623157E308) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_overflow -POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] -PREHOOK: query: insert into test_overflow values (127, -128, 32767, -32768, 2147483647, -2147483648, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_overflow -POSTHOOK: query: insert into test_overflow values (127, -128, 32767, -32768, 2147483647, -2147483648, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_overflow -POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] -PREHOOK: query: insert into test_overflow values (64, 65, 32767, -32768, 1073741824, 1073741825, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test_overflow -POSTHOOK: query: insert into test_overflow values (64, 65, 32767, -32768, 1073741824, 1073741825, 9223372036854775807, -9223372036854775808, 3.4028234663852886E38, 1.401298464324817E-45, 1.7976931348623157E308, 4.9E-324) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test_overflow -POSTHOOK: Lineage: test_overflow.cbigint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cbigint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cdouble2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cfloat2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.cint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.csmallint2 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint1 SCRIPT [] -POSTHOOK: Lineage: test_overflow.ctinyint2 SCRIPT [] -PREHOOK: query: select * from test_overflow order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select * from test_overflow order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 -64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -PREHOOK: query: explain vectorization expression -select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((cint1 - 2) > 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColGreaterLongScalar(col 13:int, val 0)(children: LongColSubtractLongScalarChecked(col 4:int, val 2) -> 13:int) - predicate: ((cint1 - 2) > 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint1 (type: int), (cint1 - 2) (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [4, 14] - selectExpressions: LongColSubtractLongScalarChecked(col 4:int, val 2) -> 14:int - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --2147483648 2147483646 -1073741824 1073741822 -2147483647 2147483645 -PREHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1, (cint1-2) from test_overflow where (cint1 - 2) > 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --2147483648 2147483646 -1073741824 1073741822 -2147483647 2147483645 -PREHOOK: query: explain vectorization expression -select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((cint2 + 2) < 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColLessLongScalar(col 13:int, val 0)(children: LongColAddLongScalarChecked(col 5:int, val 2) -> 13:int) - predicate: ((cint2 + 2) < 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint2 (type: int), (cint2 + 2) (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5, 14] - selectExpressions: LongColAddLongScalarChecked(col 5:int, val 2) -> 14:int - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --2147483648 -2147483646 -2147483647 -2147483647 -PREHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint2, (cint2+2) from test_overflow where (cint2 + 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --2147483648 -2147483646 -2147483647 -2147483647 -PREHOOK: query: explain vectorization expression -select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((cint2 * 2) < 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColLessLongScalar(col 13:int, val 0)(children: LongColMultiplyLongScalarChecked(col 5:int, val 2) -> 13:int) - predicate: ((cint2 * 2) < 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint2 (type: int), (cint2 * 2) (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5, 14] - selectExpressions: LongColMultiplyLongScalarChecked(col 5:int, val 2) -> 14:int - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -1073741825 -2147483646 -2147483647 -2 -PREHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint2, (cint2 * 2) from test_overflow where (cint2 * 2) < 0 order by cint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -1073741825 -2147483646 -2147483647 -2 -PREHOOK: query: explain vectorization expression -select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((ctinyint1 - 2Y) > 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColGreaterLongScalar(col 13:tinyint, val 0)(children: LongColSubtractLongScalarChecked(col 0:tinyint, val 2) -> 13:tinyint) - predicate: ((ctinyint1 - 2Y) > 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint1 (type: tinyint), (ctinyint1 - 2Y) (type: tinyint) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 14] - selectExpressions: LongColSubtractLongScalarChecked(col 0:tinyint, val 2) -> 14:tinyint - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: tinyint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: tinyint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 126 -64 62 -127 125 -PREHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select ctinyint1, (ctinyint1-2Y) from test_overflow where (ctinyint1 - 2Y) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 126 -64 62 -127 125 -PREHOOK: query: explain vectorization expression -select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((ctinyint2 + 2Y) < 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColLessLongScalar(col 13:tinyint, val 0)(children: LongColAddLongScalarChecked(col 1:tinyint, val 2) -> 13:tinyint) - predicate: ((ctinyint2 + 2Y) < 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint2 (type: tinyint), (ctinyint2 + 2) (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [1, 14] - selectExpressions: LongColAddLongScalarChecked(col 1:tinyint, val 2) -> 14:int - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 -126 -127 129 -PREHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select ctinyint2, (ctinyint2 + 2) from test_overflow where (ctinyint2 + 2Y) < 0 order by ctinyint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 -126 -127 129 -PREHOOK: query: explain vectorization expression -select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((csmallint2 * 2S) < 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColLessLongScalar(col 13:smallint, val 0)(children: LongColMultiplyLongScalarChecked(col 3:smallint, val 2) -> 13:smallint) - predicate: ((csmallint2 * 2S) < 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: csmallint2 (type: smallint), (csmallint2 * 2) (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [3, 14] - selectExpressions: LongColMultiplyLongScalarChecked(col 3:smallint, val 2) -> 14:int - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: smallint) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -32767 65534 -PREHOOK: query: explain vectorization expression -select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: false - enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((csmallint2 * 2S) < 0) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((csmallint2 * 2S) < 0) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: csmallint2 (type: smallint), (csmallint2 * 2) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: smallint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select csmallint2, csmallint2 * 2 from test_overflow where (csmallint2 * 2S) < 0 order by csmallint2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -32767 65534 -PREHOOK: query: create table parquettable (t1 tinyint, t2 tinyint, i1 int, i2 int) stored as parquet -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@parquettable -POSTHOOK: query: create table parquettable (t1 tinyint, t2 tinyint, i1 int, i2 int) stored as parquet -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@parquettable -PREHOOK: query: insert into parquettable values (-104, 25,2147483647, 10), (-112, 24, -2147483648, 10), (54, 9, 2147483647, -50) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@parquettable -POSTHOOK: query: insert into parquettable values (-104, 25,2147483647, 10), (-112, 24, -2147483648, 10), (54, 9, 2147483647, -50) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@parquettable -POSTHOOK: Lineage: parquettable.i1 SCRIPT [] -POSTHOOK: Lineage: parquettable.i2 SCRIPT [] -POSTHOOK: Lineage: parquettable.t1 SCRIPT [] -POSTHOOK: Lineage: parquettable.t2 SCRIPT [] -PREHOOK: query: explain vectorization expression select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: false - enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquettable - filterExpr: ((t1 - t2) < 50) (type: boolean) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((t1 - t2) < 50) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 - t2) (type: tinyint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: tinyint) - null sort order: z - sort order: - - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: tinyint) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), KEY.reducesinkkey0 (type: tinyint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -54 9 45 -PREHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: select t1, t2, (t1-t2) as diff from parquettable where (t1-t2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -54 9 45 -PREHOOK: query: explain vectorization expression select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquettable - filterExpr: ((i1 - i2) < 50) (type: boolean) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColLessLongScalar(col 5:int, val 50)(children: LongColSubtractLongColumnChecked(col 2:int, col 3:int) -> 5:int) - predicate: ((i1 - i2) < 50) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: i1 (type: int), i2 (type: int), (i1 - i2) (type: int) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [2, 3, 6] - selectExpressions: LongColSubtractLongColumnChecked(col 2:int, col 3:int) -> 6:int - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - null sort order: z - sort order: - - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -2147483647 -50 -2147483599 -PREHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -PREHOOK: type: QUERY -PREHOOK: Input: default@parquettable -#### A masked pattern was here #### -POSTHOOK: query: select i1, i2, (i1-i2) as diff from parquettable where (i1-i2) < 50 order by diff desc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquettable -#### A masked pattern was here #### -2147483647 -50 -2147483599 -PREHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PREHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PREHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1 from test_overflow where -cint1 >= 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PREHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cfloat1 from test_overflow where -cfloat1 >= 0 order by cfloat1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PREHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -2147483647 0 -PREHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select cint1, 2*cint2 from test_overflow where 2*cint2 >= 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -2147483647 0 -PREHOOK: query: select 2147483648 from test_overflow -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select 2147483648 from test_overflow -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -2147483648 -2147483648 -2147483648 -PREHOOK: query: select 2147483648 from test_overflow -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select 2147483648 from test_overflow -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -2147483648 -2147483648 -2147483648 -PREHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 -127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -PREHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select * from test_overflow where pmod(cint1, 1073741825) > 0 order by cint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 -127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -PREHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 -64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -PREHOOK: query: explain vectorization expression select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test_overflow - filterExpr: ((csmallint1 pmod 16385S) > 0) (type: boolean) - Statistics: Num rows: 3 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColGreaterLongScalar(col 13:smallint, val 0)(children: PosModLongToLong(col 2, divisor 16385) -> 13:smallint) - predicate: ((csmallint1 pmod 16385S) > 0) (type: boolean) - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint1 (type: tinyint), ctinyint2 (type: tinyint), csmallint1 (type: smallint), csmallint2 (type: smallint), cint1 (type: int), cint2 (type: int), cbigint1 (type: bigint), cbigint2 (type: bigint), cfloat1 (type: float), cfloat2 (type: float), cdouble1 (type: double), cdouble2 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: tinyint), _col2 (type: smallint), _col3 (type: smallint), _col4 (type: int), _col5 (type: int), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: float), _col9 (type: float), _col10 (type: double), _col11 (type: double) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: smallint), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: bigint), VALUE._col7 (type: float), VALUE._col8 (type: float), VALUE._col9 (type: double), VALUE._col10 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -PREHOOK: type: QUERY -PREHOOK: Input: default@test_overflow -#### A masked pattern was here #### -POSTHOOK: query: select * from test_overflow where pmod(csmallint1, 16385S) > 0 order by ctinyint1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test_overflow -#### A masked pattern was here #### --128 127 -32768 32767 -2147483648 2147483647 -9223372036854775808 9223372036854775807 1.4E-45 3.4028235E38 4.9E-324 1.7976931348623157E308 -64 65 32767 -32768 1073741824 1073741825 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 -127 -128 32767 -32768 2147483647 -2147483648 9223372036854775807 -9223372036854775808 3.4028235E38 1.4E-45 1.7976931348623157E308 4.9E-324 diff --git ql/src/test/results/clientpositive/vectorization_offset_limit.q.out ql/src/test/results/clientpositive/vectorization_offset_limit.q.out deleted file mode 100644 index 7b587a0266..0000000000 --- ql/src/test/results/clientpositive/vectorization_offset_limit.q.out +++ /dev/null @@ -1,174 +0,0 @@ -WARNING: Comparing a bigint and a double may result in a loss of precision. -PREHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) - Statistics: Num rows: 12288 Data size: 183488 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) - Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint), cdouble (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Offset of rows: 3 - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - - Stage: Stage-0 - Fetch Operator - limit: 2 - Processor Tree: - ListSink - -WARNING: Comparing a bigint and a double may result in a loss of precision. -PREHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 3,2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### --1887561756 10361.0 --1887561756 -8881.0 -PREHOOK: query: explain vectorization expression -select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - filterExpr: ctinyint is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 146796 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:tinyint) - predicate: ctinyint is not null (type: boolean) - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 5, 1] - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col2 (type: smallint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), VALUE._col0 (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Offset of rows: 10 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 3 - Processor Tree: - ListSink - -PREHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 10,3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### --64 -7196.0 -7196 --64 -6907.0 -6907 --64 -4803.0 -4803 diff --git ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out deleted file mode 100644 index 9015f31a0a..0000000000 --- ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out +++ /dev/null @@ -1,661 +0,0 @@ -PREHOOK: query: DROP TABLE IF EXISTS parquet_types_staging_n0 -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS parquet_types_staging_n0 -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE parquet_types_staging_n0 ( - cint int, - ctinyint tinyint, - csmallint smallint, - cfloat float, - cdouble double, - cstring1 string, - t timestamp, - cchar char(5), - cvarchar varchar(10), - cbinary string, - m1 map, - l1 array, - st1 struct, - d date -) ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -COLLECTION ITEMS TERMINATED BY ',' -MAP KEYS TERMINATED BY ':' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@parquet_types_staging_n0 -POSTHOOK: query: CREATE TABLE parquet_types_staging_n0 ( - cint int, - ctinyint tinyint, - csmallint smallint, - cfloat float, - cdouble double, - cstring1 string, - t timestamp, - cchar char(5), - cvarchar varchar(10), - cbinary string, - m1 map, - l1 array, - st1 struct, - d date -) ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -COLLECTION ITEMS TERMINATED BY ',' -MAP KEYS TERMINATED BY ':' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@parquet_types_staging_n0 -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging_n0 -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@parquet_types_staging_n0 -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging_n0 -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@parquet_types_staging_n0 -PREHOOK: query: DROP TABLE IF EXISTS parquet_project_test -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE IF EXISTS parquet_project_test -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE parquet_project_test( -cint int, -m1 map -) STORED AS PARQUET -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@parquet_project_test -POSTHOOK: query: CREATE TABLE parquet_project_test( -cint int, -m1 map -) STORED AS PARQUET -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@parquet_project_test -PREHOOK: query: insert into parquet_project_test -select ctinyint, map("color","red") from parquet_types_staging_n0 -where ctinyint = 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_types_staging_n0 -PREHOOK: Output: default@parquet_project_test -POSTHOOK: query: insert into parquet_project_test -select ctinyint, map("color","red") from parquet_types_staging_n0 -where ctinyint = 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_types_staging_n0 -POSTHOOK: Output: default@parquet_project_test -POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] -POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] -PREHOOK: query: insert into parquet_project_test -select ctinyint, map("color","green") from parquet_types_staging_n0 -where ctinyint = 2 -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_types_staging_n0 -PREHOOK: Output: default@parquet_project_test -POSTHOOK: query: insert into parquet_project_test -select ctinyint, map("color","green") from parquet_types_staging_n0 -where ctinyint = 2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_types_staging_n0 -POSTHOOK: Output: default@parquet_project_test -POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] -POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] -PREHOOK: query: insert into parquet_project_test -select ctinyint, map("color","blue") from parquet_types_staging_n0 -where ctinyint = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_types_staging_n0 -PREHOOK: Output: default@parquet_project_test -POSTHOOK: query: insert into parquet_project_test -select ctinyint, map("color","blue") from parquet_types_staging_n0 -where ctinyint = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_types_staging_n0 -POSTHOOK: Output: default@parquet_project_test -POSTHOOK: Lineage: parquet_project_test.cint EXPRESSION [] -POSTHOOK: Lineage: parquet_project_test.m1 EXPRESSION [] -PREHOOK: query: explain vectorization select * from parquet_project_test -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select * from parquet_project_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_project_test - Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int), m1 (type: map) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 22 Data size: 20328 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type MAP not enabled (map) since hive.vectorized.complex.types.enabled IS false - vectorized: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select * from parquet_project_test -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: select * from parquet_project_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -1 {"color":"red"} -2 {"color":"green"} -2 {"color":"green"} -2 {"color":"green"} -2 {"color":"green"} -2 {"color":"green"} -2 {"color":"green"} -2 {"color":"green"} -3 {"color":"blue"} -3 {"color":"blue"} -3 {"color":"blue"} -3 {"color":"blue"} -3 {"color":"blue"} -3 {"color":"blue"} -3 {"color":"blue"} -PREHOOK: query: explain vectorization select count(*) from parquet_project_test -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select count(*) from parquet_project_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_project_test - Statistics: Num rows: 22 Data size: 553 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 22 Data size: 553 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from parquet_project_test -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from parquet_project_test -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -22 -PREHOOK: query: explain vectorization select cint, count(*) from parquet_project_test -group by cint -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select cint, count(*) from parquet_project_test -group by cint -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_project_test - Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: cint - Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: cint (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 22 Data size: 88 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 11 Data size: 44 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cint, count(*) from parquet_project_test -group by cint -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: select cint, count(*) from parquet_project_test -group by cint -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -1 8 -2 7 -3 7 -PREHOOK: query: explain vectorization select m1["color"], count(*) from parquet_project_test -group by m1["color"] -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select m1["color"], count(*) from parquet_project_test -group by m1["color"] -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_project_test - Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: m1['color'] (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 22 Data size: 20240 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing complex type MAP not enabled (map) since hive.vectorized.complex.types.enabled IS false - vectorized: false - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 10120 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 11 Data size: 10120 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select m1["color"], count(*) from parquet_project_test -group by m1["color"] -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -POSTHOOK: query: select m1["color"], count(*) from parquet_project_test -group by m1["color"] -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_project_test -#### A masked pattern was here #### -blue 7 -green 7 -red 8 -PREHOOK: query: create table if not exists parquet_nullsplit(key string, val string) partitioned by (len string) -stored as parquet -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@parquet_nullsplit -POSTHOOK: query: create table if not exists parquet_nullsplit(key string, val string) partitioned by (len string) -stored as parquet -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@parquet_nullsplit -PREHOOK: query: insert into table parquet_nullsplit partition(len='1') -values ('one', 'red') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@parquet_nullsplit@len=1 -POSTHOOK: query: insert into table parquet_nullsplit partition(len='1') -values ('one', 'red') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@parquet_nullsplit@len=1 -POSTHOOK: Lineage: parquet_nullsplit PARTITION(len=1).key SCRIPT [] -POSTHOOK: Lineage: parquet_nullsplit PARTITION(len=1).val SCRIPT [] -PREHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_nullsplit -PREHOOK: Input: default@parquet_nullsplit@len=1 -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_nullsplit -POSTHOOK: Input: default@parquet_nullsplit@len=1 -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_nullsplit - filterExpr: (len = '1') (type: boolean) - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from parquet_nullsplit where len = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_nullsplit -PREHOOK: Input: default@parquet_nullsplit@len=1 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from parquet_nullsplit where len = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_nullsplit -POSTHOOK: Input: default@parquet_nullsplit@len=1 -#### A masked pattern was here #### -1 -PREHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '99' -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_nullsplit -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization select count(*) from parquet_nullsplit where len = '99' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_nullsplit -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: parquet_nullsplit - filterExpr: (len = '99') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (len = '99') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - inputFormatFeatureSupport: [] - featureSupportInUse: [] - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from parquet_nullsplit where len = '99' -PREHOOK: type: QUERY -PREHOOK: Input: default@parquet_nullsplit -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from parquet_nullsplit where len = '99' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@parquet_nullsplit -#### A masked pattern was here #### -0 -PREHOOK: query: drop table parquet_nullsplit -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@parquet_nullsplit -PREHOOK: Output: default@parquet_nullsplit -POSTHOOK: query: drop table parquet_nullsplit -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@parquet_nullsplit -POSTHOOK: Output: default@parquet_nullsplit -PREHOOK: query: drop table parquet_project_test -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@parquet_project_test -PREHOOK: Output: default@parquet_project_test -POSTHOOK: query: drop table parquet_project_test -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@parquet_project_test -POSTHOOK: Output: default@parquet_project_test -PREHOOK: query: drop table parquet_types_staging_n0 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@parquet_types_staging_n0 -PREHOOK: Output: default@parquet_types_staging_n0 -POSTHOOK: query: drop table parquet_types_staging_n0 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@parquet_types_staging_n0 -POSTHOOK: Output: default@parquet_types_staging_n0 diff --git ql/src/test/results/clientpositive/vectorized_join46_mr.q.out ql/src/test/results/clientpositive/vectorized_join46_mr.q.out deleted file mode 100644 index e9f696345e..0000000000 --- ql/src/test/results/clientpositive/vectorized_join46_mr.q.out +++ /dev/null @@ -1,2224 +0,0 @@ -PREHOOK: query: CREATE TABLE test1 (key INT, value INT, col_1 STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test1 -POSTHOOK: query: CREATE TABLE test1 (key INT, value INT, col_1 STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test1 -PREHOOK: query: INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'), - (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test1 -POSTHOOK: query: INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'), - (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test1 -POSTHOOK: Lineage: test1.col_1 SCRIPT [] -POSTHOOK: Lineage: test1.key SCRIPT [] -POSTHOOK: Lineage: test1.value SCRIPT [] -col1 col2 col3 -PREHOOK: query: CREATE TABLE test2 (key INT, value INT, col_2 STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test2 -POSTHOOK: query: CREATE TABLE test2 (key INT, value INT, col_2 STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test2 -PREHOOK: query: INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'), - (104, 3, 'Fli'), (105, NULL, 'None') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test2 -POSTHOOK: query: INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'), - (104, 3, 'Fli'), (105, NULL, 'None') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test2 -POSTHOOK: Lineage: test2.col_2 SCRIPT [] -POSTHOOK: Lineage: test2.key SCRIPT [] -POSTHOOK: Lineage: test2.value SCRIPT [] -col1 col2 col3 -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator - native: true - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - Map Join Vectorization: - bigTableKeyExpressions: col 1:int - className: VectorMapJoinOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob NULL NULL NULL -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None NULL NULL NULL -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND test1.key between 100 and 102 - AND test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND test1.key between 100 and 102 - AND test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - filterExpr: (value is not null and key BETWEEN 100 AND 102) (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value is not null and key BETWEEN 100 AND 102) (type: boolean) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - Map Join Vectorization: - bigTableKeyExpressions: col 1:int - className: VectorMapJoinOuterFilteredOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator - native: true - Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 8 Data size: 1049 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND test1.key between 100 and 102 - AND test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND test1.key between 100 and 102 - AND test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob NULL NULL NULL -101 2 Car 102 2 Del -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat NULL NULL NULL -NULL NULL None NULL NULL NULL -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102 - AND test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102 - AND test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - filter predicates: - 0 {_col3} - 1 - keys: - 0 - 1 - Map Join Vectorization: - className: VectorMapJoinOuterFilteredOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, Outer Join has keys IS false - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator - native: true - Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 6 Data size: 1142 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102 - AND test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102 - AND test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -101 2 Car 102 2 Del -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat NULL NULL NULL -NULL NULL None NULL NULL NULL -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value AND true) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value AND true) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:test1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:test1 - TableScan - alias: test1 - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator - native: true - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - Map Join Vectorization: - bigTableKeyExpressions: col 1:int - className: VectorMapJoinOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value AND true) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value AND true) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL NULL 104 3 Fli -NULL NULL NULL 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - selectExpressions: LongColumnBetween(col 0:int, left 100, right 102) -> 4:boolean - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - filter predicates: - 0 {_col3} - 1 - keys: - 0 - 1 - Map Join Vectorization: - className: VectorMapJoinOuterFilteredOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, Outer Join has keys IS false - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator - native: true - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat NULL NULL NULL -NULL NULL None NULL NULL NULL -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {((_col1 = _col5) or _col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None 102 2 Del -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {((_col1 = _col5) or _col3)} - Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None NULL NULL NULL -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None 102 2 Del -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_1:test2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_1:test2 - TableScan - alias: test2 - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 LEFT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob NULL NULL NULL -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat 102 2 Del -NULL NULL None NULL NULL NULL -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:test1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:test1 - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None 102 2 Del -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:test1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:test1 - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:test1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:test1 - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {((_col1 = _col4) or _col6)} - Statistics: Num rows: 24 Data size: 4676 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL NULL 104 3 Fli -NULL NULL NULL 105 NULL None -NULL NULL None 102 2 Del -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:test1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:test1 - TableScan - alias: test1 - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 7 Data size: 1265 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true - inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: MAPJOIN operator: Non-equi joins not supported - vectorized: false - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 RIGHT OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -99 2 Mat 102 2 Del -NULL NULL NULL 104 3 Fli -NULL NULL NULL 105 NULL None -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Map Vectorization: - enabled: false - enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102 - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None 102 2 Del -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Map Vectorization: - enabled: false - enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test1.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -100 1 Bob 103 2 Ema -100 1 Bob 104 3 Fli -100 1 Bob 105 NULL None -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -101 2 Car 104 3 Fli -101 2 Car 105 NULL None -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL None NULL NULL NULL -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Map Vectorization: - enabled: false - enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 4580 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - OR test2.key between 100 and 102) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob 102 2 Del -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None 102 2 Del -99 0 Alice 102 2 Del -99 2 Mat 102 2 Del -99 2 Mat 103 2 Ema -NULL NULL NULL 104 3 Fli -NULL NULL NULL 105 NULL None -NULL NULL None 102 2 Del -PREHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION OPERATOR -SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) - Map Vectorization: - enabled: false - enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} - Statistics: Num rows: 13 Data size: 1711 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 1711 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -PREHOOK: type: QUERY -PREHOOK: Input: default@test1 -PREHOOK: Input: default@test2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * -FROM test1 FULL OUTER JOIN test2 -ON (test1.value=test2.value - AND (test1.key between 100 and 102 - OR test2.key between 100 and 102)) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test1 -POSTHOOK: Input: default@test2 -#### A masked pattern was here #### -test1.key test1.value test1.col_1 test2.key test2.value test2.col_2 -100 1 Bob NULL NULL NULL -101 2 Car 102 2 Del -101 2 Car 103 2 Ema -98 NULL None NULL NULL NULL -99 0 Alice NULL NULL NULL -99 2 Mat 102 2 Del -NULL NULL NULL 104 3 Fli -NULL NULL NULL 105 NULL None -NULL NULL None NULL NULL NULL diff --git ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out deleted file mode 100644 index dc0043114f..0000000000 --- ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out +++ /dev/null @@ -1,187 +0,0 @@ -PREHOOK: query: create temporary table x (a int) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@x -POSTHOOK: query: create temporary table x (a int) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@x -PREHOOK: query: create temporary table y (b int) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@y -POSTHOOK: query: create temporary table y (b int) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@y -PREHOOK: query: insert into x values(1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@x -POSTHOOK: query: insert into x values(1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@x -POSTHOOK: Lineage: x.a SCRIPT [] -PREHOOK: query: insert into y values(1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@y -POSTHOOK: query: insert into y values(1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@y -POSTHOOK: Lineage: y.b SCRIPT [] -PREHOOK: query: explain vectorization expression -select count(1) from x, y where a = b -PREHOOK: type: QUERY -PREHOOK: Input: default@x -PREHOOK: Input: default@y -#### A masked pattern was here #### -POSTHOOK: query: explain vectorization expression -select count(1) from x, y where a = b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@x -POSTHOOK: Input: default@y -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:x - TableScan - alias: x - filterExpr: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: y - filterExpr: b is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - TableScan Vectorization: - native: true - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - predicate: b is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: b (type: int) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0] - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Map Join Vectorization: - bigTableKeyExpressions: col 0:int - className: VectorMapJoinOperator - native: false - nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Fast Hash Table and No Hybrid Hash Join IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - Group By Vectorization: - aggregators: VectorUDAFCountStar(*) -> bigint - className: VectorGroupByOperator - groupByMode: HASH - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [0] - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkOperator - native: false - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - Local Work: - Map Reduce Local Work - Reduce Vectorization: - enabled: false - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true - enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(1) from x, y where a = b -PREHOOK: type: QUERY -PREHOOK: Input: default@x -PREHOOK: Input: default@y -#### A masked pattern was here #### -POSTHOOK: query: select count(1) from x, y where a = b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@x -POSTHOOK: Input: default@y -#### A masked pattern was here #### -1 diff --git ql/src/test/results/clientpositive/windowing_gby2.q.out ql/src/test/results/clientpositive/windowing_gby2.q.out deleted file mode 100644 index edc5f9b6a3..0000000000 --- ql/src/test/results/clientpositive/windowing_gby2.q.out +++ /dev/null @@ -1,694 +0,0 @@ -PREHOOK: query: explain -select rank() over (order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by ws.key -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: explain -select rank() over (order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by ws.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ws - Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int - Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(c_int) - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 0 (type: int), _col1 (type: bigint) - null sort order: az - sort order: ++ - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col1: bigint - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col1 ASC NULLS LAST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col1 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: rank_window_0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select rank() over (order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by ws.key -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: select rank() over (order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by ws.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -1 -1 -1 -4 -4 -6 -7 -PREHOOK: query: explain -select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by cast(ws.key as int) -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: explain -select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by cast(ws.key as int) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ws - Statistics: Num rows: 20 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), c_int (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 3306 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col1), sum(_col2) - keys: _col0 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: bigint) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: string, _col2: bigint - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 ASC NULLS LAST - partition by: _col1 - raw input shape: - window functions: - window function definition - alias: avg_window_0 - arguments: _col0 - name: avg - window function: GenericUDAFAverageEvaluatorDouble - window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: avg_window_0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by cast(ws.key as int) -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: select avg(cast(ws.key as int)) over (partition by min(ws.value) order by sum(ws.c_int)) as return_rank -from cbo_t3 ws -group by cast(ws.key as int) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -NULL -1.0 -2.0 -3.0 -PREHOOK: query: explain -select rank () over(partition by key order by sum(c_int - c_float) desc) , -dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), -percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) -from cbo_t3 -group by key, value -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: explain -select rank () over(partition by key order by sum(c_int - c_float) desc) , -dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), -percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) -from cbo_t3 -group by key, value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cbo_t3 - Statistics: Num rows: 20 Data size: 3382 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), (UDFToFloat(c_int) - c_float) (type: float), (UDFToDouble(c_float) / UDFToDouble(c_int)) (type: double), c_int (type: int), ((UDFToDouble(c_float) / UDFToDouble(c_int)) - UDFToDouble(c_int)) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 20 Data size: 3382 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col2), sum(_col3), max(_col4), sum(_col5) - keys: _col0 (type: string), _col1 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: double), _col3 (type: double), _col4 (type: int), _col5 (type: double) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), max(VALUE._col2), sum(VALUE._col3) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: double) - null sort order: az - sort order: +- - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col3 (type: double), _col4 (type: int), _col5 (type: double) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: int), VALUE._col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: double, _col3: double, _col4: int, _col5: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 DESC NULLS LAST - partition by: _col0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col2 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: rank_window_0 (type: int), _col1 (type: string), _col3 (type: double), _col4 (type: int), _col5 (type: double) - outputColumnNames: rank_window_0, _col1, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: lower(_col1) (type: string), _col3 (type: double) - null sort order: az - sort order: ++ - Map-reduce partition columns: lower(_col1) (type: string) - Statistics: Num rows: 10 Data size: 1980 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: rank_window_0 (type: int), _col1 (type: string), _col4 (type: int), _col5 (type: double) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col2 (type: string), KEY.reducesinkkey1 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: double) - outputColumnNames: _col0, _col2, _col4, _col5, _col6 - Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col2: string, _col4: double, _col5: int, _col6: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col4 ASC NULLS LAST - partition by: lower(_col2) - raw input shape: - window functions: - window function definition - alias: dense_rank_window_1 - arguments: _col4 - name: dense_rank - window function: GenericUDAFDenseRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: dense_rank_window_1 (type: int), _col0 (type: int), _col5 (type: int), _col6 (type: double) - outputColumnNames: dense_rank_window_1, _col0, _col5, _col6 - Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col5 (type: int), _col6 (type: double) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: dense_rank_window_1 (type: int), _col0 (type: int) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col6, _col7 - Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: int, _col1: int, _col6: int, _col7: double - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col7 ASC NULLS LAST - partition by: _col6 - raw input shape: - window functions: - window function definition - alias: percent_rank_window_2 - arguments: _col7 - name: percent_rank - window function: GenericUDAFPercentRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: int), percent_rank_window_2 (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select rank () over(partition by key order by sum(c_int - c_float) desc) , -dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), -percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) -from cbo_t3 -group by key, value -PREHOOK: type: QUERY -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: select rank () over(partition by key order by sum(c_int - c_float) desc) , -dense_rank () over(partition by lower(value) order by sum(c_float/c_int) asc), -percent_rank () over(partition by max(c_int) order by sum((c_float/c_int) - c_int) asc) -from cbo_t3 -group by key, value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -1 1 0.0 -1 1 0.0 -1 1 0.0 -1 1 0.0 -1 1 0.0 -1 1 0.0 -1 1 0.0 -PREHOOK: query: explain -select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank -from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 -group by ws.c_boolean -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: explain -select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank -from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 -group by ws.c_boolean -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: ws - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1767 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string), c_int (type: int), c_boolean (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1581 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: boolean) - TableScan - alias: wr - filterExpr: cstring1 is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 899146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: cstring1 is not null (type: boolean) - Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int), cstring1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 9174 Data size: 671296 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 27 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col3), sum(_col1) - keys: _col2 (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: boolean) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: bigint), _col2 (type: bigint) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 0 (type: int), (UDFToDouble(_col1) / UDFToDouble(_col2)) (type: double) - null sort order: az - sort order: ++ - Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col1 (type: bigint), VALUE._col2 (type: bigint) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col1: bigint, _col2: bigint - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: (UDFToDouble(_col1) / UDFToDouble(_col2)) ASC NULLS LAST - partition by: 0 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: (UDFToDouble(_col1) / UDFToDouble(_col2)) - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: rank_window_0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank -from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 -group by ws.c_boolean -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -PREHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### -POSTHOOK: query: select rank() over (order by sum(wr.cint)/sum(ws.c_int)) as return_rank -from cbo_t3 ws join alltypesorc wr on ws.value = wr.cstring1 -group by ws.c_boolean -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -POSTHOOK: Input: default@cbo_t3 -#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/windowing_udaf.q.out ql/src/test/results/clientpositive/windowing_udaf.q.out deleted file mode 100644 index af69e9d87f..0000000000 --- ql/src/test/results/clientpositive/windowing_udaf.q.out +++ /dev/null @@ -1,598 +0,0 @@ -PREHOOK: query: drop table over10k_n4 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table over10k_n4 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table over10k_n4( - t tinyint, - si smallint, - i int, - b bigint, - f float, - d double, - bo boolean, - s string, - ts timestamp, - `dec` decimal, - bin binary) - row format delimited - fields terminated by '|' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@over10k_n4 -POSTHOOK: query: create table over10k_n4( - t tinyint, - si smallint, - i int, - b bigint, - f float, - d double, - bo boolean, - s string, - ts timestamp, - `dec` decimal, - bin binary) - row format delimited - fields terminated by '|' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@over10k_n4 -PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k_n4 -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@over10k_n4 -POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k_n4 -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@over10k_n4 -PREHOOK: query: select s, min(i) over (partition by s) from over10k_n4 limit 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -POSTHOOK: query: select s, min(i) over (partition by s) from over10k_n4 limit 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice allen 65545 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice brown 65569 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice carson 65545 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice davidson 65544 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice ellison 65537 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice falkner 65596 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice garcia 65573 -alice hernandez 65543 -alice hernandez 65543 -alice hernandez 65543 -alice hernandez 65543 -alice hernandez 65543 -PREHOOK: query: select s, avg(f) over (partition by si order by s) from over10k_n4 limit 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -POSTHOOK: query: select s, avg(f) over (partition by si order by s) from over10k_n4 limit 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -alice ellison 38.15999984741211 -alice ovid 57.26999855041504 -alice robinson 61.9533322652181 -calvin johnson 48.489999294281006 -calvin young 57.602000045776364 -david davidson 60.36000029246012 -ethan falkner 53.59000028882708 -ethan thompson 50.917500138282776 -fred hernandez 45.63555569118924 -fred quirinius 45.169000244140626 -gabriella falkner 44.16181841763583 -holly xylophone 42.55583349863688 -irene polk 46.413846236008865 -nick zipper 49.762142998831614 -oscar steinbeck 50.098000081380206 -quinn hernandez 52.19624996185303 -quinn quirinius 51.01882351146025 -rachel ellison 48.684444427490234 -rachel ovid 47.271052611501595 -sarah brown 49.46350011825562 -sarah white 48.66238103594099 -tom allen 48.59454545107755 -ulysses white 48.839565194171406 -ulysses young 47.483333349227905 -victor davidson 45.79800001144409 -victor hernandez 46.39076920656058 -victor johnson 45.437777748814334 -victor robinson 46.12964274202074 -victor thompson 47.65344815418638 -wendy garcia 47.7129998366038 -wendy nixon 49.234193478861165 -xavier ovid 49.311874970793724 -xavier quirinius 50.531212185368396 -xavier van buren 51.34705878706539 -zach nixon 50.55771423067365 -zach underhill 50.889999959203934 -zach white 51.42081073812536 -alice underhill 17.329999923706055 -alice xylophone 57.66499996185303 -david brown 65.36333401997884 -ethan king 68.81000089645386 -ethan polk 73.13600120544433 -ethan underhill 76.83833408355713 -fred brown 79.57714380536761 -fred zipper 72.68500089645386 -gabriella ellison 68.5955564710829 -holly robinson 69.57500076293945 -holly white 67.0809097290039 -irene young 63.535833994547524 -jessica laertes 62.61846219576322 -katie garcia 59.8014292035784 -katie ichabod 59.41533381144206 -luke falkner 57.51437544822693 -luke quirinius 56.508823843563306 -mike underhill 54.72500027550591 -nick davidson 54.44210544385408 -nick quirinius 55.77850008010864 -nick zipper 57.73428571791876 -oscar laertes 55.53090910478072 -oscar thompson 53.169130449709684 -oscar young 53.24833333492279 -priscilla laertes 53.86000003814697 -priscilla miller 52.379230792705826 -priscilla thompson 50.80222225189209 -rachel robinson 51.162142923900056 -sarah hernandez 49.505517301888304 -ulysses young 48.93933340708415 -victor xylophone 49.34774195763372 -wendy brown 49.7678125500679 -xavier brown 49.81303030071837 -yuri brown 49.74735299278708 -yuri nixon 49.86571431841169 -zach hernandez 49.37763158898605 -zach hernandez 49.37763158898605 -zach hernandez 49.37763158898605 -zach underhill 50.13846162649301 -zach zipper 51.274500131607056 -alice nixon 31.255000591278076 -alice nixon 31.255000591278076 -bob falkner 51.356666247049965 -calvin davidson 39.96249973773956 -calvin johnson 32.491999769210814 -calvin steinbeck 29.613333185513813 -david garcia 34.99428541319711 -fred king 40.516249507665634 -gabriella young 39.50888853602939 -holly johnson 40.59399974346161 -irene quirinius 43.26727249405601 -jessica falkner 42.47249974807104 -jessica ichabod 42.86076903343201 -jessica laertes 44.109285541943144 -katie allen 47.71466666857402 -luke ovid 50.293125078082085 -mike underhill 50.38294123200809 -nick falkner 52.106666922569275 -nick king 54.01842131112751 -nick xylophone 54.5635001540184 -oscar van buren 55.146666697093416 -priscilla underhill 56.049545580690555 -priscilla zipper 55.24625018239021 -PREHOOK: query: select s, avg(i) over (partition by t, b order by s) from over10k_n4 limit 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -POSTHOOK: query: select s, avg(i) over (partition by t, b order by s) from over10k_n4 limit 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -oscar miller 65773.0 -bob underhill 65735.0 -tom polk 65743.5 -wendy robinson 65622.0 -alice king 65570.0 -ethan steinbeck 65720.0 -quinn polk 65695.5 -bob ellison 65671.0 -mike xylophone 65610.0 -alice king 65563.0 -ulysses robinson 65540.0 -victor hernandez 65755.0 -xavier ovid 65769.0 -oscar garcia 65712.0 -sarah zipper 65766.0 -wendy xylophone 65670.0 -gabriella brown 65751.0 -irene young 65715.0 -sarah miller 65680.33333333333 -nick nixon 65696.0 -tom steinbeck 65608.0 -calvin white 65548.0 -oscar johnson 65650.0 -holly thompson 65550.0 -david falkner 65698.0 -ulysses steinbeck 65562.0 -victor allen 65658.0 -rachel thompson 65733.0 -jessica johnson 65589.0 -alice falkner 65597.0 -priscilla quirinius 65646.0 -ulysses carson 65714.5 -victor thompson 65728.33333333333 -ulysses ellison 65640.0 -bob johnson 65731.0 -quinn ellison 65736.0 -bob polk 65776.0 -victor xylophone 65697.0 -jessica white 65566.0 -holly nixon 65565.0 -oscar johnson 65671.0 -irene ellison 65696.0 -priscilla zipper 65669.0 -fred ellison 65548.0 -nick robinson 65566.0 -sarah robinson 65605.0 -tom hernandez 65575.0 -alice xylophone 65661.0 -sarah xylophone 65618.0 -priscilla laertes 65696.0 -holly polk 65710.0 -katie ichabod 65577.0 -holly underhill 65553.0 -alice davidson 65676.0 -ethan underhill 65706.5 -bob van buren 65654.0 -jessica king 65644.0 -mike allen 65551.0 -luke garcia 65569.0 -tom hernandez 65785.0 -mike white 65705.0 -yuri xylophone 65667.0 -holly steinbeck 65613.0 -ulysses quirinius 65617.0 -mike polk 65704.0 -bob underhill 65627.0 -xavier garcia 65662.0 -david garcia 65684.0 -oscar davidson 65646.0 -zach young 65573.0 -zach thompson 65716.0 -calvin ichabod 65635.0 -alice allen 65557.0 -david hernandez 65667.0 -david carson 65702.0 -luke king 65629.0 -mike king 65543.0 -rachel davidson 65728.0 -luke ichabod 65757.0 -luke young 65696.0 -victor robinson 65571.0 -wendy quirinius 65616.0 -jessica miller 65763.0 -bob ichabod 65733.0 -bob falkner 65595.0 -irene underhill 65634.0 -zach zipper 65588.0 -alice underhill 65790.0 -alice xylophone 65715.0 -ethan nixon 65782.0 -bob van buren 65582.0 -katie ovid 65613.0 -fred king 65712.0 -yuri hernandez 65657.5 -nick falkner 65604.0 -david laertes 65612.0 -sarah steinbeck 65655.0 -alice carson 65565.0 -katie thompson 65582.5 -oscar van buren 65595.0 -PREHOOK: query: select max(i) over w from over10k_n4 window w as (partition by f) limit 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -POSTHOOK: query: select max(i) over w from over10k_n4 window w as (partition by f) limit 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -65774 -65774 -65786 -65786 -65786 -65738 -65738 -65695 -65584 -65774 -65774 -65743 -65743 -65743 -65654 -65654 -65750 -65743 -65722 -65722 -65712 -65696 -65696 -65763 -65763 -65733 -65733 -65764 -65718 -65604 -65754 -65754 -65683 -65683 -65683 -65776 -65776 -65776 -65744 -65744 -65595 -65738 -65670 -65658 -65777 -65777 -65704 -65710 -65710 -65612 -65600 -65600 -65600 -65780 -65780 -65752 -65752 -65752 -65580 -65787 -65787 -65787 -65618 -65778 -65592 -65733 -65645 -65788 -65617 -65736 -65782 -65676 -65676 -65676 -65648 -65782 -65782 -65782 -65782 -65604 -65756 -65751 -65546 -65771 -65756 -65756 -65756 -65791 -65555 -65646 -65726 -65726 -65540 -65759 -65573 -65630 -65545 -65785 -65590 -65740 -PREHOOK: query: select s, avg(d) over (partition by t order by f) from over10k_n4 limit 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -POSTHOOK: query: select s, avg(d) over (partition by t order by f) from over10k_n4 limit 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@over10k_n4 -#### A masked pattern was here #### -rachel thompson 11.96 -oscar davidson 19.57 -mike polk 22.643333333333334 -alice king 24.672500000000003 -luke garcia 26.828000000000003 -luke young 29.035 -bob van buren 25.648571428571433 -ulysses carson 24.712500000000002 -bob ellison 23.742222222222225 -holly underhill 24.645000000000003 -ulysses quirinius 22.532727272727275 -oscar miller 22.8175 -david garcia 21.680769230769233 -zach thompson 20.301428571428573 -wendy xylophone 21.272000000000002 -ethan underhill 21.608750000000004 -bob underhill 22.438823529411767 -zach young 22.509444444444444 -alice falkner 23.94578947368421 -quinn ellison 23.249000000000002 -bob polk 22.426190476190477 -priscilla zipper 22.000909090909094 -priscilla laertes 22.931739130434785 -ulysses steinbeck 23.458750000000006 -bob johnson 23.428800000000006 -nick robinson 22.78461538461539 -wendy quirinius 21.941481481481485 -ulysses robinson 22.58214285714286 -alice xylophone 22.361724137931038 -xavier garcia 22.62333333333334 -xavier ovid 23.0241935483871 -calvin ichabod 22.315312500000005 -ethan steinbeck 22.842121212121217 -david carson 22.20176470588236 -sarah xylophone 21.63685714285715 -victor allen 21.37055555555556 -victor hernandez 21.334324324324328 -oscar garcia 22.05552631578948 -irene ellison 21.707692307692312 -david falkner 22.395500000000006 -jessica johnson 22.20048780487805 -victor thompson 22.123095238095242 -calvin white 22.377441860465122 -oscar johnson 22.59863636363637 -luke king 22.241777777777784 -tom polk 22.498043478260875 -luke ichabod 22.960638297872347 -sarah miller 22.94958333333334 -bob falkner 22.61897959183674 -ulysses ellison 22.326000000000008 -quinn polk 22.701960784313734 -victor robinson 22.73153846153847 -mike white 22.57283018867925 -victor xylophone 22.240370370370375 -alice king 22.59054545454546 -jessica white 22.885357142857146 -priscilla quirinius 23.08105263157895 -holly steinbeck 23.473620689655174 -irene young 23.81813559322034 -yuri xylophone 24.0955 -holly nixon 23.776721311475413 -wendy robinson 23.53016129032258 -alice davidson 23.656984126984128 -nick nixon 23.438281250000003 -bob underhill 23.425076923076926 -bob ichabod 23.349242424242426 -mike king 23.258805970149258 -alice allen 23.349705882352943 -gabriella brown 23.015362318840584 -tom steinbeck 23.387857142857143 -tom hernandez 23.389859154929578 -david hernandez 23.7025 -rachel davidson 24.02356164383562 -mike xylophone 24.119324324324328 -holly polk 23.964133333333336 -mike allen 24.262500000000003 -sarah robinson 24.570779220779226 -holly thompson 24.5474358974359 -jessica miller 24.368987341772154 -katie ichabod 24.476375000000004 -irene underhill 24.525308641975315 -jessica king 24.227195121951222 -oscar johnson 24.24807228915663 -fred ellison 24.47404761904762 -tom hernandez 24.313647058823534 -sarah zipper 24.095813953488374 -ulysses polk 20.12 -nick falkner 33.435 -david falkner 33.43 -wendy ellison 32.5325 -ethan polk 29.6 -tom ichabod 26.988333333333333 -ethan quirinius 25.58857142857143 -alice underhill 23.74125 -mike laertes 22.99888888888889 -zach zipper 20.915 -zach zipper 19.822727272727274 -ulysses king 18.273333333333333 -bob king 17.664615384615384 -luke carson 18.02785714285714 -PREHOOK: query: select key, max(value) over - (order by key rows between 10 preceding and 20 following) -from src1 where length(key) > 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src1 -#### A masked pattern was here #### -POSTHOOK: query: select key, max(value) over - (order by key rows between 10 preceding and 20 following) -from src1 where length(key) > 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src1 -#### A masked pattern was here ####