commit 9d70af8cd7579a0abc8fc63545c6b6f303319929 Author: Janaki Lahorani Date: Mon Jun 18 12:02:33 2018 -0700 HIVE-19940: PPD of predicates with deterministic UDFs with RBO diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java index b01a9bad7e65ac20f466b26a0287e9afdb74e37d..1c662d7cfb5b4a999d66b93ded266c8d8860d25a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java @@ -104,7 +104,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return false; } else { if (exp instanceof ExprNodeGenericFuncDesc) { - isCandidate = false; + if (isDeterministic((ExprNodeGenericFuncDesc) exp)) { + isCandidate = true; + } else { + isCandidate = false; + } } if (exp instanceof ExprNodeColumnDesc && ci == null) { ExprNodeColumnDesc column = (ExprNodeColumnDesc)exp; @@ -135,6 +139,30 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } + /** + * + * @param funcDesc function descriptor + * @return true if the function is deterministic false otherwise + */ + public static boolean isDeterministic(ExprNodeGenericFuncDesc funcDesc) { + if (FunctionRegistry.isConsistentWithinQuery(funcDesc.getGenericUDF())) { + // check whether the children or deterministic + for (ExprNodeDesc exprNodeDesc : funcDesc.getChildren()) { + if (exprNodeDesc instanceof ExprNodeGenericFuncDesc) { + if (!isDeterministic((ExprNodeGenericFuncDesc) exprNodeDesc)) { + // some child is not deterministic - return false + return false; + } + } + } + // all children are deterministic - return true + return true; + } + + // function is not deterministic - return false + return false; + } + /** * FieldExprProcessor. * diff --git ql/src/test/queries/clientpositive/ppd_deterministic_expr.q ql/src/test/queries/clientpositive/ppd_deterministic_expr.q new file mode 100644 index 0000000000000000000000000000000000000000..47c88493c623b5aa4eb924efce03de9e1b1173fb --- /dev/null +++ ql/src/test/queries/clientpositive/ppd_deterministic_expr.q @@ -0,0 +1,143 @@ +set hive.auto.convert.join=false; +set hive.optimize.index.filter=true; +set hive.cbo.enable=false; + +CREATE TABLE `testb`( + `cola` string COMMENT '', + `colb` string COMMENT '', + `colc` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) + +STORED AS AVRO; + +CREATE TABLE `testa`( + `col1` string COMMENT '', + `col2` string COMMENT '', + `col3` string COMMENT '', + `col4` string COMMENT '', + `col5` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) +STORED AS AVRO; + +insert into testA partition (part1='US', part2='ABC', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd'); + +insert into testA partition (part1='UK', part2='DEF', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd'); + +insert into testA partition (part1='US', part2='DEF', part3='200') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd'); + +insert into testA partition (part1='CA', part2='ABC', part3='300') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd'); + +insert into testB partition (part1='CA', part2='ABC', part3='300') +values ('600', '700', 'abc'), ('601', '701', 'abcd'); + +insert into testB partition (part1='CA', part2='ABC', part3='400') +values ( '600', '700', 'abc'), ( '601', '701', 'abcd'); + +insert into testB partition (part1='UK', part2='PQR', part3='500') +values ('600', '700', 'abc'), ('601', '701', 'abcd'); + +insert into testB partition (part1='US', part2='DEF', part3='200') +values ( '600', '700', 'abc'), ('601', '701', 'abcd'); + +insert into testB partition (part1='US', part2='PQR', part3='123') +values ( '600', '700', 'abc'), ('601', '701', 'abcd'); + +-- views with deterministic functions +create view viewDeterministicUDFA partitioned on (vpart1, vpart2, vpart3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testa +where part1 in ('US', 'CA'); + +create view viewDeterministicUDFB partitioned on (vpart1, vpart2, vpart3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testb +where part1 in ('US', 'CA'); + +-- views without function reference +create view viewNoUDFA partitioned on (part1, part2, part3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + part1, + part2, + part3 + from testa +where part1 in ('US', 'CA'); + +create view viewNoUDFB partitioned on (part1, part2, part3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + part1, + part2, + part3 + from testb +where part1 in ('US', 'CA'); + +-- query referencing deterministic functions +explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200'; + +select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200'; + +-- query with views referencing no udfs +explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200'; + +select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200'; diff --git ql/src/test/queries/clientpositive/ppd_udf_col.q ql/src/test/queries/clientpositive/ppd_udf_col.q index 9c7d4fdc20dad955469c3baa688595c2961cad6b..ac2f861f4f3c6401b99d034810c3ed705e47e26b 100644 --- ql/src/test/queries/clientpositive/ppd_udf_col.q +++ ql/src/test/queries/clientpositive/ppd_udf_col.q @@ -48,3 +48,51 @@ EXPLAIN SELECT key,randum123, v10 FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a WHERE a.v10 <= 200; + +set hive.cbo.enable=false; + +EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1; + +EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20; + +EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3; + +EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200; + +set hive.ppd.remove.duplicatefilters=false; + +EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1; + +EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20; + +EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3; + +EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200; diff --git ql/src/test/results/clientpositive/llap/check_constraint.q.out ql/src/test/results/clientpositive/llap/check_constraint.q.out index 411b4a6ea0c53353b11cc0500ccf59049238ccae..190a2572c07165a55119788dcb0b104ab0670e2d 100644 --- ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -169,11 +169,11 @@ POSTHOOK: Lineage: table1_n0.j SCRIPT [] PREHOOK: query: SELECT * from table1_n0 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from table1_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 100 true 5 23.4 700 PREHOOK: query: DROP TABLE table1_n0 PREHOOK: type: DROPTABLE @@ -298,11 +298,11 @@ POSTHOOK: Lineage: table2_n0.i SCRIPT [] PREHOOK: query: select * from table2_n0 PREHOOK: type: QUERY PREHOOK: Input: default@table2_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from table2_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@table2_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 8 PREHOOK: query: Drop table table2_n0 PREHOOK: type: DROPTABLE @@ -536,11 +536,11 @@ POSTHOOK: Lineage: tmulti.username SCRIPT [] PREHOOK: query: Select * from tmulti PREHOOK: type: QUERY PREHOOK: Input: default@tmulti -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: Select * from tmulti POSTHOOK: type: QUERY POSTHOOK: Input: default@tmulti -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### hive.apache.com user1 48 NULL PREHOOK: query: truncate table tmulti PREHOOK: type: TRUNCATETABLE @@ -695,11 +695,11 @@ POSTHOOK: Lineage: tmulti.username SCRIPT [] PREHOOK: query: Select * from tmulti PREHOOK: type: QUERY PREHOOK: Input: default@tmulti -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: Select * from tmulti POSTHOOK: type: QUERY POSTHOOK: Input: default@tmulti -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### hive.apache.com user1 48 NULL PREHOOK: query: Drop table tmulti PREHOOK: type: DROPTABLE @@ -840,11 +840,11 @@ POSTHOOK: Lineage: tcase.username SCRIPT [] PREHOOK: query: Select * from tcase PREHOOK: type: QUERY PREHOOK: Input: default@tcase -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: Select * from tcase POSTHOOK: type: QUERY POSTHOOK: Input: default@tcase -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### hive.apache.com user1 NULL 48 PREHOOK: query: Drop table tcase PREHOOK: type: DROPTABLE @@ -981,11 +981,11 @@ POSTHOOK: Lineage: tcast.url SCRIPT [] PREHOOK: query: SELECT * from tcast PREHOOK: type: QUERY PREHOOK: Input: default@tcast -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from tcast POSTHOOK: type: QUERY POSTHOOK: Input: default@tcast -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### www.google.com 100 0.5 PREHOOK: query: EXPLAIN INSERT INTO tcast(url, price) values('www.yahoo.com', 0.5) PREHOOK: type: QUERY @@ -1063,11 +1063,11 @@ POSTHOOK: Lineage: tcast.url SCRIPT [] PREHOOK: query: SELECT * FROM tcast PREHOOK: type: QUERY PREHOOK: Input: default@tcast -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM tcast POSTHOOK: type: QUERY POSTHOOK: Input: default@tcast -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### www.google.com 100 0.5 www.yahoo.com NULL 0.5 PREHOOK: query: DROP TABLE tcast @@ -1217,11 +1217,11 @@ POSTHOOK: Lineage: texpr.i SCRIPT [] PREHOOK: query: SELECT * from texpr PREHOOK: type: QUERY PREHOOK: Input: default@texpr -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from texpr POSTHOOK: type: QUERY POSTHOOK: Input: default@texpr -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 3 3.4 5.6 true PREHOOK: query: DROP TABLE texpr PREHOOK: type: DROPTABLE @@ -1615,11 +1615,11 @@ POSTHOOK: Lineage: acid_uami_n0.vc EXPRESSION [(src)s1.FieldSchema(name:value, t PREHOOK: query: select * from acid_uami_n0 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 2 2.00 val_2 9 9.00 val_9 8 8.00 val_8 @@ -1761,11 +1761,11 @@ POSTHOOK: Lineage: acid_uami_n0.vc EXPRESSION [(src)src.FieldSchema(name:value, PREHOOK: query: select * from acid_uami_n0 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 113 113.00 val_113 105 105.00 val_105 10 10.00 val_10 @@ -1820,20 +1820,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key < 10) and enforce_constraint((CAST( key AS decimal(5,2)) is not null and (CAST( key AS decimal(5,2)) >= CAST( UDFToInteger(key) AS decimal(5,2))) is not false))) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 34362 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: enforce_constraint((_col1 is not null and (_col1 >= CAST( _col0 AS decimal(5,2))) is not false)) (type: boolean) + Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) Filter Operator predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE @@ -1903,11 +1900,11 @@ POSTHOOK: Output: default@src_multi2_n0 PREHOOK: query: select * from acid_uami_n0 order by de desc limit 15 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n0 order by de desc limit 15 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain update acid_uami_n0 set de = 893.14 where de = 103.00 or de = 119.00 PREHOOK: type: QUERY POSTHOOK: query: explain update acid_uami_n0 set de = 893.14 where de = 103.00 or de = 119.00 @@ -1988,11 +1985,11 @@ POSTHOOK: Output: default@acid_uami_n0 PREHOOK: query: select * from acid_uami_n0 order by de desc limit 15 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n0 order by de desc limit 15 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: ALTER table acid_uami_n0 drop constraint ch2 PREHOOK: type: ALTERTABLE_DROPCONSTRAINT POSTHOOK: query: ALTER table acid_uami_n0 drop constraint ch2 @@ -2077,11 +2074,11 @@ POSTHOOK: Output: default@acid_uami_n0 PREHOOK: query: select * from acid_uami_n0 order by vc limit 15 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n0 order by vc limit 15 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: DROP TABLE acid_uami_n0 PREHOOK: type: DROPTABLE PREHOOK: Input: default@acid_uami_n0 @@ -2987,11 +2984,11 @@ POSTHOOK: Lineage: numericdatatype.b SCRIPT [] PREHOOK: query: SELECT * from numericDataType PREHOOK: type: QUERY PREHOOK: Input: default@numericdatatype -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from numericDataType POSTHOOK: type: QUERY POSTHOOK: Input: default@numericdatatype -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 127 456 PREHOOK: query: DROP TABLE numericDataType PREHOOK: type: DROPTABLE @@ -3214,11 +3211,11 @@ POSTHOOK: Lineage: tcheck.b SCRIPT [] PREHOOK: query: SELECT * FROM tcheck PREHOOK: type: QUERY PREHOOK: Input: default@tcheck -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM tcheck POSTHOOK: type: QUERY POSTHOOK: Input: default@tcheck -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### NULL 4 PREHOOK: query: DROP TABLE tcheck PREHOOK: type: DROPTABLE @@ -3327,11 +3324,11 @@ POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key EXPRESSION [(src)src.FieldS PREHOOK: query: select key from src order by value limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key from src order by value limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 @@ -3339,12 +3336,12 @@ PREHOOK: query: select * from part_mm PREHOOK: type: QUERY PREHOOK: Input: default@part_mm PREHOOK: Input: default@part_mm@key_mm=455 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@part_mm POSTHOOK: Input: default@part_mm@key_mm=455 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 98 455 98 455 97 455 @@ -3541,11 +3538,11 @@ POSTHOOK: Lineage: tbl1_n1.b SCRIPT [] PREHOOK: query: select * from tbl1_n1 PREHOOK: type: QUERY PREHOOK: Input: default@tbl1_n1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tbl1_n1 POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl1_n1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### a 69 PREHOOK: query: ALTER TABLE tbl1_n1 add constraint chk2 CHECK (b < 100) PREHOOK: type: ALTERTABLE_ADDCONSTRAINT diff --git ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out index 5a3f51956a9b345e5e48fe82832abea9a071eb98..0fe51fbee17ebc04073eb529a25397f1a0905ebc 100644 --- ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out +++ ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -118,11 +118,11 @@ POSTHOOK: Lineage: table1_n7.c SCRIPT [] PREHOOK: query: SELECT * FROM table1_n7 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n7 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### not null constraint PREHOOK: query: explain insert into table1_n7(a,c) values('1','2') PREHOOK: type: QUERY @@ -338,11 +338,11 @@ POSTHOOK: Lineage: table1_n7.c SIMPLE [(src)src.FieldSchema(name:value, type:str PREHOOK: query: SELECT * FROM table1_n7 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n7 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 val_0 0 val_0 val_0 0 val_0 val_0 @@ -950,11 +950,11 @@ POSTHOOK: Lineage: table1_n7.c SIMPLE [(src)src.FieldSchema(name:value, type:str PREHOOK: query: SELECT * FROM table1_n7 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n7 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 val_0 0 val_0 val_0 0 val_0 val_0 @@ -1560,11 +1560,11 @@ POSTHOOK: Lineage: table1_n7.c SIMPLE [(src)src.FieldSchema(name:value, type:str PREHOOK: query: SELECT * FROM table1_n7 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n7 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 val_0 0 0 val_0 0 0 val_0 @@ -2592,11 +2592,11 @@ POSTHOOK: Lineage: acid_uami_n1.vc SCRIPT [] PREHOOK: query: select * from acid_uami_n1 PREHOOK: type: QUERY PREHOOK: Input: default@acid_uami_n1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from acid_uami_n1 POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_uami_n1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 109.23 mary had a little lamb 6553 923.19 its fleece was white as snow PREHOOK: query: --insert into select @@ -3424,20 +3424,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key < 10) and enforce_constraint((UDFToInteger(key) is not null and value is not null))) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 34362 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: enforce_constraint((_col0 is not null and _col2 is not null)) (type: boolean) + Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: decimal(5,2)), _col2 (type: string) Filter Operator predicate: ((key < 20) and (key > 10) and enforce_constraint(value is not null)) (type: boolean) Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE @@ -3878,7 +3875,7 @@ PREHOOK: Input: default@tablepartitioned@p1=484/p2=3 PREHOOK: Input: default@tablepartitioned@p1=86/p2=3 PREHOOK: Input: default@tablepartitioned@p1=98/p2=3 PREHOOK: Input: default@tablepartitioned@p1=today/p2=10 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tablePartitioned POSTHOOK: type: QUERY POSTHOOK: Input: default@tablepartitioned @@ -3893,7 +3890,7 @@ POSTHOOK: Input: default@tablepartitioned@p1=484/p2=3 POSTHOOK: Input: default@tablepartitioned@p1=86/p2=3 POSTHOOK: Input: default@tablepartitioned@p1=98/p2=3 POSTHOOK: Input: default@tablepartitioned@p1=today/p2=10 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 165 val_165 val_165 165 3 238 val_238 val_238 238 3 255 val_255 val_255 255 3 @@ -5986,11 +5983,11 @@ POSTHOOK: Lineage: part_mm_n1 PARTITION(key_mm=455).key EXPRESSION [(src)src.Fie PREHOOK: query: select key from src order by value limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key from src order by value limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 @@ -5998,12 +5995,12 @@ PREHOOK: query: select * from part_mm_n1 PREHOOK: type: QUERY PREHOOK: Input: default@part_mm_n1 PREHOOK: Input: default@part_mm_n1@key_mm=455 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part_mm_n1 POSTHOOK: type: QUERY POSTHOOK: Input: default@part_mm_n1 POSTHOOK: Input: default@part_mm_n1@key_mm=455 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 455 0 455 0 455 diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 8f3912e1420457665b4e94bcc480fcb707fe8e2f..be7eb1d7309b920b4091a2fb97850f60690d2d4d 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -96,13 +96,13 @@ from src where src.key in (select key from src s1 where s1.key > '9') PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src where src.key in (select key from src s1 where s1.key > '9') POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 val_90 90 val_90 90 val_90 @@ -223,7 +223,7 @@ where b.key in ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src b where b.key in @@ -233,7 +233,7 @@ where b.key in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 val_90 90 val_90 90 val_90 @@ -409,7 +409,7 @@ part where part.p_size in ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name, p_size from part where part.p_size in @@ -419,7 +419,7 @@ part where part.p_size in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### almond antique medium spring khaki 6 almond antique salmon chartreuse burlywood 6 PREHOOK: query: explain @@ -596,7 +596,7 @@ from part b where b.p_size in ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_mfgr, p_name, p_size from part b where b.p_size in (select min(p_size) @@ -605,7 +605,7 @@ from part b where b.p_size in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#2 almond aquamarine midnight light salmon 2 @@ -721,7 +721,7 @@ where b.key in ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src b where b.key in @@ -731,7 +731,7 @@ where b.key in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 val_90 90 val_90 90 val_90 @@ -878,7 +878,7 @@ where b.key in ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src b where b.key in @@ -888,7 +888,7 @@ where b.key in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 val_90 90 val_90 90 val_90 @@ -1003,25 +1003,25 @@ STAGE PLANS: PREHOOK: query: select * from src b where b.key in (select distinct key from src a where a.value > b.value) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src b where b.key in (select distinct key from src a where a.value > b.value) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: select p_mfgr, p_name, p_size from part where part.p_size in (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_mfgr, p_name, p_size from part where part.p_size in (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#2 almond aquamarine midnight light salmon 2 @@ -1186,14 +1186,14 @@ where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') PREHOOK: type: QUERY PREHOOK: Input: default@lineitem -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 108570 8571 115118 7630 115209 7721 @@ -1210,14 +1210,14 @@ where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) PREHOOK: type: QUERY PREHOOK: Input: default@lineitem -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 108570 8571 4297 1798 PREHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) @@ -1392,12 +1392,12 @@ PREHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@lineitem PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### NULL PREHOOK: query: explain select * from part where p_brand <> 'Brand#14' AND p_size IN (select (p_size) from part p where p.p_type = part.p_type group by p_size) AND p_size <> 340 PREHOOK: type: QUERY @@ -1504,11 +1504,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_brand <> 'Brand#14' AND p_size IN (select (p_size) from part p where p.p_type = part.p_type group by p_size) AND p_size <> 340 PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_brand <> 'Brand#14' AND p_size IN (select (p_size) from part p where p.p_type = part.p_type group by p_size) AND p_size <> 340 POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car @@ -1662,11 +1662,11 @@ STAGE PLANS: PREHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h @@ -1806,11 +1806,11 @@ STAGE PLANS: PREHOOK: query: select * from part where (p_partkey*p_size) IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_partkey*p_size) IN (select min(p_partkey) from part group by p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s PREHOOK: query: explain select count(*) as c from part as e where p_size + 100 IN (select p_partkey from part where p_name = e.p_name) PREHOOK: type: QUERY @@ -1918,11 +1918,11 @@ STAGE PLANS: PREHOOK: query: select count(*) as c from part as e where p_size + 100 IN (select p_partkey from part where p_name = e.p_name) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select count(*) as c from part as e where p_size + 100 IN (select p_partkey from part where p_name = e.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 PREHOOK: query: explain select * from part where floor(p_retailprice) IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY @@ -2057,11 +2057,11 @@ STAGE PLANS: PREHOOK: query: select * from part where floor(p_retailprice) IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where floor(p_retailprice) IN (select floor(min(p_retailprice)) from part group by p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car @@ -2234,11 +2234,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_name IN (select p_name from part p where p.p_size = part.p_size AND part.p_size + 121150 = p.p_partkey ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_name IN (select p_name from part p where p.p_size = part.p_size AND part.p_size + 121150 = p.p_partkey ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h PREHOOK: query: explain select * from part where p_name IN (select p_name from part p where p.p_size = part.p_size AND part.p_partkey= p.p_partkey ) @@ -2331,11 +2331,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_name IN (select p_name from part p where p.p_size = part.p_size AND part.p_partkey= p.p_partkey ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_name IN (select p_name from part p where p.p_size = part.p_size AND part.p_partkey= p.p_partkey ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car @@ -2452,11 +2452,11 @@ STAGE PLANS: PREHOOK: query: select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type IN (select p_type from part where part.p_brand = fpart.brand) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type IN (select p_type from part where part.p_brand = fpart.brand) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### almond antique blue firebrick mint almond antique burnished rose metallic almond antique burnished rose metallic @@ -2634,11 +2634,11 @@ STAGE PLANS: PREHOOK: query: select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type IN (select p_type from part where (part.p_size+1) = fpart.size) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type IN (select p_type from part where (part.p_size+1) = fpart.size) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### almond antique blue firebrick mint almond antique burnished rose metallic almond antique burnished rose metallic @@ -2854,11 +2854,11 @@ STAGE PLANS: PREHOOK: query: select key, count(*) from src where value IN (select value from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key, count(*) from src where value IN (select value from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 3 119 3 128 3 @@ -3078,11 +3078,11 @@ STAGE PLANS: PREHOOK: query: select key, count(*) from src where value IN (select value from src sc where sc.key = src.key ) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key, count(*) from src where value IN (select value from src sc where sc.key = src.key ) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 3 119 3 128 3 @@ -3256,11 +3256,11 @@ STAGE PLANS: PREHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) order by p_brand POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h @@ -3417,11 +3417,11 @@ STAGE PLANS: PREHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) order by p_brand limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) IN (select min(p_size) from part group by p_type) order by p_brand limit 4 POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle @@ -3546,12 +3546,12 @@ PREHOOK: query: select * from src where key IN (select p_name from part UNION AL PREHOOK: type: QUERY PREHOOK: Input: default@part PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src where key IN (select p_name from part UNION ALL select p_brand from part) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain select p_mfgr, b.p_name, p_size from part b where b.p_name in (select p_name from (select p_mfgr, p_name, p_size as r from part) a where r < 10 and b.p_mfgr = a.p_mfgr ) order by p_mfgr,p_size PREHOOK: type: QUERY @@ -3662,12 +3662,12 @@ PREHOOK: query: select p_mfgr, b.p_name, p_size from part b where b.p_name in (select p_name from (select p_mfgr, p_name, p_size as r from part) a where r < 10 and b.p_mfgr = a.p_mfgr ) order by p_mfgr,p_size PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_mfgr, b.p_name, p_size from part b where b.p_name in (select p_name from (select p_mfgr, p_name, p_size as r from part) a where r < 10 and b.p_mfgr = a.p_mfgr ) order by p_mfgr,p_size POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique salmon chartreuse burlywood 6 @@ -3809,11 +3809,11 @@ STAGE PLANS: PREHOOK: query: select p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 110592 112398 @@ -3976,12 +3976,12 @@ PREHOOK: query: select p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 110592 112398 @@ -4276,12 +4276,12 @@ PREHOOK: query: select p_partkey from (select p_size, p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size)) subq PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_partkey from (select p_size, p_partkey from part where p_name in (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size)) subq POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 110592 112398 @@ -4475,11 +4475,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_size IN (select count(*) from part pp where pp.p_type = part.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size IN (select count(*) from part pp where pp.p_type = part.p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s @@ -4575,23 +4575,19 @@ STAGE PLANS: predicate: (_col1 is not null and _col2 is not null) (type: boolean) Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: (_col1 / _col2) (type: double), _col0 (type: int) + expressions: _col0 (type: int), (_col1 / _col2) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: double) + Group By Operator + keys: _col0 (type: int), _col1 (type: double) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int), _col1 (type: double) - mode: hash - outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: double) Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: double) - Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -4602,11 +4598,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_size in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car @@ -4801,11 +4797,11 @@ Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reduc PREHOOK: query: select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref @@ -5031,11 +5027,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_size NOT IN (select count(*) from part pp where pp.p_type = part.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size NOT IN (select count(*) from part pp where pp.p_type = part.p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car @@ -5290,11 +5286,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: create table t_n22(i int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -5430,12 +5426,12 @@ PREHOOK: query: select * from t_n22 where i IN (select count(*) from tempty_n2) PREHOOK: type: QUERY PREHOOK: Input: default@t_n22 PREHOOK: Input: default@tempty_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from t_n22 where i IN (select count(*) from tempty_n2) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n22 POSTHOOK: Input: default@tempty_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 PREHOOK: query: drop table t_n22 PREHOOK: type: DROPTABLE @@ -5466,40 +5462,40 @@ PREHOOK: query: select * from part where p_size IN (select i from tempty_n2) PREHOOK: type: QUERY PREHOOK: Input: default@part PREHOOK: Input: default@tempty_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size IN (select i from tempty_n2) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@tempty_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: select * from tnull_n2 where i IN (select i from tempty_n2) PREHOOK: type: QUERY PREHOOK: Input: default@tempty_n2 PREHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tnull_n2 where i IN (select i from tempty_n2) POSTHOOK: type: QUERY POSTHOOK: Input: default@tempty_n2 POSTHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: select * from part where p_size IN (select i from tnull_n2) PREHOOK: type: QUERY PREHOOK: Input: default@part PREHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size IN (select i from tnull_n2) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: select * from tnull_n2 where i IN (select i from tnull_n2) PREHOOK: type: QUERY PREHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tnull_n2 where i IN (select i from tnull_n2) POSTHOOK: type: QUERY POSTHOOK: Input: default@tnull_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: drop table tempty_n2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@tempty_n2 @@ -5660,12 +5656,12 @@ PREHOOK: query: select * from t_n22 where i IN (select sum(i) from tt_n2 where t PREHOOK: type: QUERY PREHOOK: Input: default@t_n22 PREHOOK: Input: default@tt_n2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from t_n22 where i IN (select sum(i) from tt_n2 where tt_n2.j = t_n22.j) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n22 POSTHOOK: Input: default@tt_n2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: drop table t_n22 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t_n22 @@ -5850,11 +5846,11 @@ Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reduc PREHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) @@ -6042,11 +6038,11 @@ Warning: Shuffle Join MERGEJOIN[71][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reduc PREHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 469ec6931fcb8b1c191fb341e937ef58f9a0a83b..4f3366aa4057f69040b6d3631570cec7a032f42f 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -164,14 +164,14 @@ where src.key not in ( select key from src s1 where s1.key > '2') order by key PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src where src.key not in ( select key from src s1 where s1.key > '2') order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 @@ -547,7 +547,7 @@ where b.p_name not in order by p_mfgr, b.p_name PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_mfgr, b.p_name, p_size from part b where b.p_name not in @@ -558,7 +558,7 @@ where b.p_name not in order by p_mfgr, b.p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Manufacturer#1 almond antique chartreuse lavender yellow 34 Manufacturer#1 almond antique salmon chartreuse burlywood 6 Manufacturer#1 almond aquamarine burnished black steel 28 @@ -818,7 +818,7 @@ part where part.p_size not in order by p_name, p_size PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name, p_size from part where part.p_size not in @@ -829,7 +829,7 @@ part where part.p_size not in order by p_name, p_size POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### almond antique blue firebrick mint 31 almond antique burnished rose metallic 2 almond antique burnished rose metallic 2 @@ -1191,7 +1191,7 @@ from part b where b.p_size not in ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_mfgr, p_name, p_size from part b where b.p_size not in (select min(p_size) @@ -1200,7 +1200,7 @@ from part b where b.p_size not in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Manufacturer#1 almond antique salmon chartreuse burlywood 6 Manufacturer#1 almond aquamarine burnished black steel 28 Manufacturer#1 almond antique chartreuse lavender yellow 34 @@ -1229,7 +1229,7 @@ where li.l_linenumber = 1 and group by li.l_partkey PREHOOK: type: QUERY PREHOOK: Input: default@lineitem -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select li.l_partkey, count(*) from lineitem li where li.l_linenumber = 1 and @@ -1237,7 +1237,7 @@ where li.l_linenumber = 1 and group by li.l_partkey POSTHOOK: type: QUERY POSTHOOK: Input: default@lineitem -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 22630 1 119477 1 450 1 @@ -1261,14 +1261,14 @@ where not src.key in ( select key from src s1 where s1.key > '2') order by key PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src where not src.key in ( select key from src s1 where s1.key > '2') order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 @@ -1416,7 +1416,7 @@ POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v POSTHOOK: Lineage: T2_v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1478,7 +1478,7 @@ STAGE PLANS: Select Operator expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 15272 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) mode: hash @@ -1570,21 +1570,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@t1_v PREHOOK: Input: default@t2_v -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@t1_v POSTHOOK: Input: default@t2_v -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain select * from part where p_brand <> 'Brand#14' AND p_size NOT IN (select (p_size*p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_brand <> 'Brand#14' AND p_size NOT IN (select (p_size*p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 @@ -1754,11 +1754,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_brand <> 'Brand#14' AND p_size NOT IN (select (p_size*p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_brand <> 'Brand#14' AND p_size NOT IN (select (p_size*p_size) from part p where p.p_type = part.p_type ) AND p_size <> 340 POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle @@ -1969,11 +1969,11 @@ Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s @@ -2164,11 +2164,11 @@ Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h @@ -2421,13 +2421,13 @@ STAGE PLANS: PREHOOK: query: select count(*) as c from part as e where p_size + 100 NOT IN (select p_partkey from part where p_name = e.p_name) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select count(*) as c from part as e where p_size + 100 NOT IN (select p_partkey from part where p_name = e.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 26 -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) @@ -2550,16 +2550,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Group By Operator - keys: _col0 (type: bigint) - mode: hash + Select Operator + expressions: _col1 (type: double) + outputColumnNames: _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: floor(_col1) (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2597,15 +2605,15 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir PREHOOK: query: explain select * from part where p_name NOT IN (select p_name from part p where p.p_size = part.p_size AND part.p_size + 121150 = p.p_partkey ) PREHOOK: type: QUERY @@ -2848,11 +2856,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_name NOT IN (select p_name from part p where p.p_size = part.p_size AND part.p_size + 121150 = p.p_partkey ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_name NOT IN (select p_name from part p where p.p_size = part.p_size AND part.p_size + 121150 = p.p_partkey ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull @@ -3034,11 +3042,11 @@ STAGE PLANS: PREHOOK: query: select * from part where p_name NOT IN (select p_name from part p where p.p_size = part.p_size AND part.p_partkey= p.p_partkey ) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where p_name NOT IN (select p_name from part p where p.p_size = part.p_size AND part.p_partkey= p.p_partkey ) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type NOT IN (select p_type+2 from part where part.p_brand = fpart.brand) PREHOOK: type: QUERY POSTHOOK: query: explain select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type NOT IN (select p_type+2 from part where part.p_brand = fpart.brand) @@ -3106,7 +3114,7 @@ STAGE PLANS: Select Operator expressions: (UDFToDouble(p_type) + 2.0D) (type: double), p_brand (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: double), _col1 (type: string) mode: hash @@ -3204,11 +3212,11 @@ STAGE PLANS: PREHOOK: query: select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type NOT IN (select p_type+2 from part where part.p_brand = fpart.brand) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name from (select p_name, p_type, p_brand as brand from part) fpart where fpart.p_type NOT IN (select p_type+2 from part where part.p_brand = fpart.brand) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type NOT IN (select p_type from part where (part.p_size+1) = fpart.size) PREHOOK: type: QUERY POSTHOOK: query: explain select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type NOT IN (select p_type from part where (part.p_size+1) = fpart.size) @@ -3223,11 +3231,12 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) + Reducer 11 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) + Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3286,7 +3295,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part - filterExpr: p_size is not null (type: boolean) + filterExpr: (p_size is not null or p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: p_size is not null (type: boolean) @@ -3305,6 +3314,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 10 @@ -3320,6 +3346,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 11 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -3454,11 +3488,11 @@ STAGE PLANS: PREHOOK: query: select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type NOT IN (select p_type from part where (part.p_size+1) = fpart.size+1) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select p_name from (select p_name, p_type, p_size+1 as size from part) fpart where fpart.p_type NOT IN (select p_type from part where (part.p_size+1) = fpart.size+1) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### almond aquamarine yellow dodger mint almond aquamarine dodger light gainsboro almond antique misty red olive @@ -3724,11 +3758,11 @@ Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 438 3 417 3 403 3 @@ -3822,7 +3856,7 @@ STAGE PLANS: Select Operator expressions: concat('v', value) (type: string), key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 135500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col1 (type: string), _col0 (type: string) mode: hash @@ -4016,11 +4050,11 @@ STAGE PLANS: PREHOOK: query: select key, count(*) from src where value NOT IN (select concat('v', value) from src sc where sc.key = src.key ) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key, count(*) from src where value NOT IN (select concat('v', value) from src sc where sc.key = src.key ) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 438 3 417 3 403 3 @@ -4239,11 +4273,11 @@ Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously @@ -4451,11 +4485,11 @@ Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously @@ -4669,12 +4703,12 @@ PREHOOK: query: select * from src where key NOT IN (select p_name from part UNIO PREHOOK: type: QUERY PREHOOK: Input: default@part PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 @@ -5402,11 +5436,11 @@ STAGE PLANS: PREHOOK: query: select count(*) as c from part as e where p_size + 100 not in ( select p_type from part where p_brand = e.p_brand) PREHOOK: type: QUERY PREHOOK: Input: default@part -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select count(*) as c from part as e where p_size + 100 not in ( select p_type from part where p_brand = e.p_brand) POSTHOOK: type: QUERY POSTHOOK: Input: default@part -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 26 PREHOOK: query: CREATE TABLE t1_n0 (c1 INT, c2 CHAR(100)) PREHOOK: type: CREATETABLE @@ -5592,12 +5626,12 @@ PREHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 PREHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n0 POSTHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: explain SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0 where t1_n0.c2=t2_n0.c1) PREHOOK: type: QUERY POSTHOOK: query: explain SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0 where t1_n0.c2=t2_n0.c1) @@ -5811,12 +5845,12 @@ PREHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0 where PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 PREHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0 where t1_n0.c2=t2_n0.c1) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n0 POSTHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### NULL 1 2 @@ -6031,12 +6065,12 @@ PREHOOK: query: select t1_n0.a from t1_n0 where t1_n0.b NOT IN (select t2_n0.a f PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 PREHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select t1_n0.a from t1_n0 where t1_n0.b NOT IN (select t2_n0.a from t2_n0 where t2_n0.b=t1_n0.a) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n0 POSTHOOK: Input: default@t2_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: drop table t1_n0 PREHOOK: type: DROPTABLE PREHOOK: Input: default@t1_n0 @@ -6284,12 +6318,12 @@ PREHOOK: query: select * from fixOb where j NOT IN (select i from t7 where t7.j= PREHOOK: type: QUERY PREHOOK: Input: default@fixob PREHOOK: Input: default@t7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) POSTHOOK: type: QUERY POSTHOOK: Input: default@fixob POSTHOOK: Input: default@t7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -1 15 PREHOOK: query: drop table t7 PREHOOK: type: DROPTABLE @@ -6511,11 +6545,11 @@ STAGE PLANS: PREHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 where t1_n0.j=t_n0.j) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 where t1_n0.j=t_n0.j) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 1 4 @@ -6668,11 +6702,11 @@ STAGE PLANS: PREHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 where t1_n0.j=t_n0.j) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 where t1_n0.j=t_n0.j) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) @@ -6822,11 +6856,11 @@ Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 4 Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product @@ -6970,11 +7004,11 @@ Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reduc PREHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### PREHOOK: query: drop table t1_n0 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1_n0 @@ -7204,7 +7238,7 @@ where b.key not in ) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from src b where b.key not in @@ -7214,7 +7248,7 @@ where b.key not in ) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 diff --git ql/src/test/results/clientpositive/masking_disablecbo_2.q.out ql/src/test/results/clientpositive/masking_disablecbo_2.q.out index fad012013058759e9a0ecf9754ec78b8b9c73d52..5a70e005e22b35aa39f3f812f3d9e444cce20f13 100644 --- ql/src/test/results/clientpositive/masking_disablecbo_2.q.out +++ ql/src/test/results/clientpositive/masking_disablecbo_2.q.out @@ -24,25 +24,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -81,25 +78,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10) and (_col0 > 0)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -135,27 +129,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and reverse(value) is not null and (UDFToInteger(key) > 0)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10) and (_col0 > 0)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0) and reverse(value) is not null) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) + value expressions: _col0 (type: int) TableScan alias: a filterExpr: key is not null (type: boolean) @@ -210,47 +198,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and UDFToDouble(UDFToInteger(key)) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10) and UDFToDouble(_col0) is not null) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and UDFToDouble(UDFToInteger(key)) is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and UDFToDouble(reverse(value)) is not null and (UDFToInteger(key) > 0)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10) and (_col0 > 0)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0) and UDFToDouble(reverse(value)) is not null) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: UDFToDouble(_col1) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col1) (type: double) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: UDFToDouble(_col1) is not null (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col1) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col1) (type: double) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) + value expressions: _col0 (type: int) Reduce Operator Tree: Join Operator condition map: @@ -288,56 +267,50 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Union + Union + Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE TableScan alias: src + filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((_col0 % 2) = 0) and (_col0 < 10) and (_col0 > 0)) (type: boolean) + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), reverse(_col1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Union + Union + Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: string) diff --git ql/src/test/results/clientpositive/perf/tez/query8.q.out ql/src/test/results/clientpositive/perf/tez/query8.q.out index d9b82b44097f8eaaba2d683fe1dddc44c5e5b412..26c7d8b71d2ba9ade8c5eee2c90ee1471e573b1c 100644 --- ql/src/test/results/clientpositive/perf/tez/query8.q.out +++ ql/src/test/results/clientpositive/perf/tez/query8.q.out @@ -234,150 +234,150 @@ Stage-0 limit:100 Stage-1 Reducer 5 vectorized - File Output Operator [FS_152] - Limit [LIM_151] (rows=100 width=88) + File Output Operator [FS_150] + Limit [LIM_149] (rows=100 width=88) Number of rows:100 - Select Operator [SEL_150] (rows=348477374 width=88) + Select Operator [SEL_148] (rows=348477374 width=88) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_149] - Group By Operator [GBY_148] (rows=348477374 width=88) + SHUFFLE [RS_147] + Group By Operator [GBY_146] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 Group By Operator [GBY_56] (rows=696954748 width=88) Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col8 - Merge Join Operator [MERGEJOIN_119] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_117] (rows=696954748 width=88) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2","_col8"] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_118] (rows=1874 width=1911) - Conds:RS_139.substr(_col0, 1, 2)=RS_142.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] + Merge Join Operator [MERGEJOIN_116] (rows=1874 width=1911) + Conds:RS_137.substr(_col0, 1, 2)=RS_140.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] <-Map 19 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_142] + SHUFFLE [RS_140] PartitionCols:substr(_col2, 1, 2) - Select Operator [SEL_141] (rows=1704 width=1910) + Select Operator [SEL_139] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_140] (rows=1704 width=1910) + Filter Operator [FIL_138] (rows=1704 width=1910) predicate:(s_store_sk is not null and substr(s_zip, 1, 2) is not null) TableScan [TS_42] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] <-Reducer 11 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_139] + SHUFFLE [RS_137] PartitionCols:substr(_col0, 1, 2) - Select Operator [SEL_138] (rows=1 width=1014) + Select Operator [SEL_136] (rows=1 width=1014) Output:["_col0"] - Filter Operator [FIL_137] (rows=1 width=1014) + Filter Operator [FIL_135] (rows=1 width=1014) predicate:(_col1 = 2L) - Group By Operator [GBY_136] (rows=6833333 width=1014) + Group By Operator [GBY_134] (rows=6833333 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Union 10 [SIMPLE_EDGE] <-Reducer 17 [CONTAINS] vectorized - Reduce Output Operator [RS_173] + Reduce Output Operator [RS_171] PartitionCols:_col0 - Group By Operator [GBY_172] (rows=13666666 width=1014) + Group By Operator [GBY_170] (rows=13666666 width=1014) Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_171] (rows=3666666 width=1014) + Group By Operator [GBY_169] (rows=3666666 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Reducer 16 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_170] + SHUFFLE [RS_168] PartitionCols:_col0 - Group By Operator [GBY_169] (rows=7333333 width=1014) + Group By Operator [GBY_167] (rows=7333333 width=1014) Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_168] (rows=7333333 width=1014) + Select Operator [SEL_166] (rows=7333333 width=1014) Output:["_col0"] - Filter Operator [FIL_167] (rows=7333333 width=1014) + Filter Operator [FIL_165] (rows=7333333 width=1014) predicate:(_col1 > 10L) - Group By Operator [GBY_166] (rows=22000000 width=1014) + Group By Operator [GBY_164] (rows=22000000 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 Group By Operator [GBY_24] (rows=44000000 width=1014) Output:["_col0","_col1"],aggregations:["count()"],keys:_col1 - Merge Join Operator [MERGEJOIN_117] (rows=44000000 width=1014) - Conds:RS_162._col0=RS_165._col0(Inner),Output:["_col1"] + Merge Join Operator [MERGEJOIN_115] (rows=44000000 width=1014) + Conds:RS_160._col0=RS_163._col0(Inner),Output:["_col1"] <-Map 14 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_162] + SHUFFLE [RS_160] PartitionCols:_col0 - Select Operator [SEL_161] (rows=40000000 width=1014) + Select Operator [SEL_159] (rows=40000000 width=1014) Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=40000000 width=1014) + Filter Operator [FIL_158] (rows=40000000 width=1014) predicate:(ca_address_sk is not null and substr(substr(ca_zip, 1, 5), 1, 2) is not null) TableScan [TS_14] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] <-Map 18 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_165] + SHUFFLE [RS_163] PartitionCols:_col0 - Select Operator [SEL_164] (rows=40000000 width=860) + Select Operator [SEL_162] (rows=40000000 width=860) Output:["_col0"] - Filter Operator [FIL_163] (rows=40000000 width=860) + Filter Operator [FIL_161] (rows=40000000 width=860) predicate:((c_preferred_cust_flag = 'Y') and c_current_addr_sk is not null) TableScan [TS_17] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_current_addr_sk","c_preferred_cust_flag"] <-Reducer 9 [CONTAINS] vectorized - Reduce Output Operator [RS_159] + Reduce Output Operator [RS_157] PartitionCols:_col0 - Group By Operator [GBY_158] (rows=13666666 width=1014) + Group By Operator [GBY_156] (rows=13666666 width=1014) Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_157] (rows=10000000 width=1014) + Group By Operator [GBY_155] (rows=10000000 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Map 8 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_156] + SHUFFLE [RS_154] PartitionCols:_col0 - Group By Operator [GBY_155] (rows=20000000 width=1014) + Group By Operator [GBY_153] (rows=20000000 width=1014) Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_154] (rows=20000000 width=1014) + Select Operator [SEL_152] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_153] (rows=20000000 width=1014) + Filter Operator [FIL_151] (rows=20000000 width=1014) predicate:((substr(ca_zip, 1, 5)) IN ('89436', '30868', '65085', '22977', '83927', '77557', '58429', '40697', '80614', '10502', '32779', '91137', '61265', '98294', '17921', '18427', '21203', '59362', '87291', '84093', '21505', '17184', '10866', '67898', '25797', '28055', '18377', '80332', '74535', '21757', '29742', '90885', '29898', '17819', '40811', '25990', '47513', '89531', '91068', '10391', '18846', '99223', '82637', '41368', '83658', '86199', '81625', '26696', '89338', '88425', '32200', '81427', '19053', '77471', '36610', '99823', '43276', '41249', '48584', '83550', '82276', '18842', '78890', '14090', '38123', '40936', '34425', '19850', '43286', '80072', '79188', '54191', '11395', '50497', '84861', '90733', '21068', '57666', '37119', '25004', '57835', '70067', '62878', '95806', '19303', '18840', '19124', '29785', '16737', '16022', '49613', '89977', '68310', '60069', '98360', '48649', '39050', '41793', '25002', '27413', '39736', '47208', '16515', '94808', '57648', '15009', '80015', '42961', '63982', '21744', '71853', '81087', '67468', '34175', '64008', '20261', '11201', '51799', '48043', '45645', '61163', '48375', '36447', '57042', '21218', '41100', '89951', '22745', '35851', '83326', '61125', '78298', '80752', '49858', '52940', '96976', '63792', '11376', '53582', '18717', '90226', '50530', '94203', '99447', '27670', '96577', '57856', '56372', '16165', '23427', '54561', '28806', '44439', '22926', '30123', '61451', '92397', '56979', '92309', '70873', '13355', '21801', '46346', '37562', '56458', '28286', '47306', '99555', '69399', '26234', '47546', '49661', '88601', '35943', '39936', '25632', '24611', '44166', '56648', '30379', '59785', '11110', '14329', '93815', '52226', '71381', '13842', '25612', '63294', '14664', '21077', '82626', '18799', '60915', '81020', '56447', '76619', '11433', '13414', '42548', '92713', '70467', '30884', '47484', '16072', '38936', '13036', '88376', '45539', '35901', '19506', '65690', '73957', '71850', '49231', '14276', '20005', '18384', '76615', '11635', '38177', '55607', '41369', '95447', '58581', '58149', '91946', '33790', '76232', '75692', '95464', '22246', '51061', '56692', '53121', '77209', '15482', '10688', '14868', '45907', '73520', '72666', '25734', '17959', '24677', '66446', '94627', '53535', '15560', '41967', '69297', '11929', '59403', '33283', '52232', '57350', '43933', '40921', '36635', '10827', '71286', '19736', '80619', '25251', '95042', '15526', '36496', '55854', '49124', '81980', '35375', '49157', '63512', '28944', '14946', '36503', '54010', '18767', '23969', '43905', '66979', '33113', '21286', '58471', '59080', '13395', '79144', '70373', '67031', '38360', '26705', '50906', '52406', '26066', '73146', '15884', '31897', '30045', '61068', '45550', '92454', '13376', '14354', '19770', '22928', '97790', '50723', '46081', '30202', '14410', '20223', '88500', '67298', '13261', '14172', '81410', '93578', '83583', '46047', '94167', '82564', '21156', '15799', '86709', '37931', '74703', '83103', '23054', '70470', '72008', '49247', '91911', '69998', '20961', '70070', '63197', '54853', '88191', '91830', '49521', '19454', '81450', '89091', '62378', '25683', '61869', '51744', '36580', '85778', '36871', '48121', '28810', '83712', '45486', '67393', '26935', '42393', '20132', '55349', '86057', '21309', '80218', '10094', '11357', '48819', '39734', '40758', '30432', '21204', '29467', '30214', '61024', '55307', '74621', '11622', '68908', '33032', '52868', '99194', '99900', '84936', '69036', '99149', '45013', '32895', '59004', '32322', '14933', '32936', '33562', '72550', '27385', '58049', '58200', '16808', '21360', '32961', '18586', '79307', '15492') and substr(substr(ca_zip, 1, 5), 1, 2) is not null) TableScan [TS_6] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_zip"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_116] (rows=633595212 width=88) - Conds:RS_147._col0=RS_130._col0(Inner),Output:["_col1","_col2"] + Merge Join Operator [MERGEJOIN_114] (rows=633595212 width=88) + Conds:RS_145._col0=RS_128._col0(Inner),Output:["_col1","_col2"] <-Map 6 [SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_130] + PARTITION_ONLY_SHUFFLE [RS_128] PartitionCols:_col0 - Select Operator [SEL_129] (rows=18262 width=1119) + Select Operator [SEL_127] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_128] (rows=18262 width=1119) + Filter Operator [FIL_126] (rows=18262 width=1119) predicate:((d_qoy = 1) and (d_year = 2002) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_147] + SHUFFLE [RS_145] PartitionCols:_col0 - Select Operator [SEL_146] (rows=575995635 width=88) + Select Operator [SEL_144] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_145] (rows=575995635 width=88) + Filter Operator [FIL_143] (rows=575995635 width=88) predicate:((ss_sold_date_sk BETWEEN DynamicValue(RS_50_date_dim_d_date_sk_min) AND DynamicValue(RS_50_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_50_date_dim_d_date_sk_bloom_filter))) and (ss_store_sk BETWEEN DynamicValue(RS_53_store_s_store_sk_min) AND DynamicValue(RS_53_store_s_store_sk_max) and in_bloom_filter(ss_store_sk, DynamicValue(RS_53_store_s_store_sk_bloom_filter))) and ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] <-Reducer 13 [BROADCAST_EDGE] vectorized - BROADCAST [RS_144] - Group By Operator [GBY_143] (rows=1 width=12) + BROADCAST [RS_142] + Group By Operator [GBY_141] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] <-Reducer 12 [CUSTOM_SIMPLE_EDGE] - SHUFFLE [RS_93] - Group By Operator [GBY_92] (rows=1 width=12) + SHUFFLE [RS_91] + Group By Operator [GBY_90] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_91] (rows=1874 width=1911) + Select Operator [SEL_89] (rows=1874 width=1911) Output:["_col0"] - Please refer to the previous Merge Join Operator [MERGEJOIN_118] + Please refer to the previous Merge Join Operator [MERGEJOIN_116] <-Reducer 7 [BROADCAST_EDGE] vectorized - BROADCAST [RS_135] - Group By Operator [GBY_134] (rows=1 width=12) + BROADCAST [RS_133] + Group By Operator [GBY_132] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] <-Map 6 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_133] - Group By Operator [GBY_132] (rows=1 width=12) + PARTITION_ONLY_SHUFFLE [RS_131] + Group By Operator [GBY_130] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_131] (rows=18262 width=1119) + Select Operator [SEL_129] (rows=18262 width=1119) Output:["_col0"] - Please refer to the previous Select Operator [SEL_129] + Please refer to the previous Select Operator [SEL_127] diff --git ql/src/test/results/clientpositive/ppd_deterministic_expr.q.out ql/src/test/results/clientpositive/ppd_deterministic_expr.q.out new file mode 100644 index 0000000000000000000000000000000000000000..b96a0e23c2418ba7f154915fbba7777e7fd3775e --- /dev/null +++ ql/src/test/results/clientpositive/ppd_deterministic_expr.q.out @@ -0,0 +1,553 @@ +PREHOOK: query: CREATE TABLE `testb`( + `cola` string COMMENT '', + `colb` string COMMENT '', + `colc` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) + +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testb +POSTHOOK: query: CREATE TABLE `testb`( + `cola` string COMMENT '', + `colb` string COMMENT '', + `colc` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) + +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testb +PREHOOK: query: CREATE TABLE `testa`( + `col1` string COMMENT '', + `col2` string COMMENT '', + `col3` string COMMENT '', + `col4` string COMMENT '', + `col5` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testa +POSTHOOK: query: CREATE TABLE `testa`( + `col1` string COMMENT '', + `col2` string COMMENT '', + `col3` string COMMENT '', + `col4` string COMMENT '', + `col5` string COMMENT '') +PARTITIONED BY ( + `part1` string, + `part2` string, + `part3` string) +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testa +PREHOOK: query: insert into testA partition (part1='US', part2='ABC', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testa@part1=US/part2=ABC/part3=123 +POSTHOOK: query: insert into testA partition (part1='US', part2='ABC', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testa@part1=US/part2=ABC/part3=123 +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=ABC,part3=123).col1 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=ABC,part3=123).col2 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=ABC,part3=123).col3 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=ABC,part3=123).col4 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=ABC,part3=123).col5 SCRIPT [] +PREHOOK: query: insert into testA partition (part1='UK', part2='DEF', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testa@part1=UK/part2=DEF/part3=123 +POSTHOOK: query: insert into testA partition (part1='UK', part2='DEF', part3='123') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testa@part1=UK/part2=DEF/part3=123 +POSTHOOK: Lineage: testa PARTITION(part1=UK,part2=DEF,part3=123).col1 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=UK,part2=DEF,part3=123).col2 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=UK,part2=DEF,part3=123).col3 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=UK,part2=DEF,part3=123).col4 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=UK,part2=DEF,part3=123).col5 SCRIPT [] +PREHOOK: query: insert into testA partition (part1='US', part2='DEF', part3='200') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testa@part1=US/part2=DEF/part3=200 +POSTHOOK: query: insert into testA partition (part1='US', part2='DEF', part3='200') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testa@part1=US/part2=DEF/part3=200 +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=DEF,part3=200).col1 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=DEF,part3=200).col2 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=DEF,part3=200).col3 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=DEF,part3=200).col4 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=US,part2=DEF,part3=200).col5 SCRIPT [] +PREHOOK: query: insert into testA partition (part1='CA', part2='ABC', part3='300') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testa@part1=CA/part2=ABC/part3=300 +POSTHOOK: query: insert into testA partition (part1='CA', part2='ABC', part3='300') +values ('12.34', '100', '200', '300', 'abc'), +('12.341', '1001', '2001', '3001', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testa@part1=CA/part2=ABC/part3=300 +POSTHOOK: Lineage: testa PARTITION(part1=CA,part2=ABC,part3=300).col1 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=CA,part2=ABC,part3=300).col2 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=CA,part2=ABC,part3=300).col3 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=CA,part2=ABC,part3=300).col4 SCRIPT [] +POSTHOOK: Lineage: testa PARTITION(part1=CA,part2=ABC,part3=300).col5 SCRIPT [] +PREHOOK: query: insert into testB partition (part1='CA', part2='ABC', part3='300') +values ('600', '700', 'abc'), ('601', '701', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testb@part1=CA/part2=ABC/part3=300 +POSTHOOK: query: insert into testB partition (part1='CA', part2='ABC', part3='300') +values ('600', '700', 'abc'), ('601', '701', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testb@part1=CA/part2=ABC/part3=300 +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=300).cola SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=300).colb SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=300).colc SCRIPT [] +PREHOOK: query: insert into testB partition (part1='CA', part2='ABC', part3='400') +values ( '600', '700', 'abc'), ( '601', '701', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testb@part1=CA/part2=ABC/part3=400 +POSTHOOK: query: insert into testB partition (part1='CA', part2='ABC', part3='400') +values ( '600', '700', 'abc'), ( '601', '701', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testb@part1=CA/part2=ABC/part3=400 +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=400).cola SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=400).colb SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=CA,part2=ABC,part3=400).colc SCRIPT [] +PREHOOK: query: insert into testB partition (part1='UK', part2='PQR', part3='500') +values ('600', '700', 'abc'), ('601', '701', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testb@part1=UK/part2=PQR/part3=500 +POSTHOOK: query: insert into testB partition (part1='UK', part2='PQR', part3='500') +values ('600', '700', 'abc'), ('601', '701', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testb@part1=UK/part2=PQR/part3=500 +POSTHOOK: Lineage: testb PARTITION(part1=UK,part2=PQR,part3=500).cola SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=UK,part2=PQR,part3=500).colb SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=UK,part2=PQR,part3=500).colc SCRIPT [] +PREHOOK: query: insert into testB partition (part1='US', part2='DEF', part3='200') +values ( '600', '700', 'abc'), ('601', '701', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testb@part1=US/part2=DEF/part3=200 +POSTHOOK: query: insert into testB partition (part1='US', part2='DEF', part3='200') +values ( '600', '700', 'abc'), ('601', '701', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testb@part1=US/part2=DEF/part3=200 +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=DEF,part3=200).cola SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=DEF,part3=200).colb SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=DEF,part3=200).colc SCRIPT [] +PREHOOK: query: insert into testB partition (part1='US', part2='PQR', part3='123') +values ( '600', '700', 'abc'), ('601', '701', 'abcd') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testb@part1=US/part2=PQR/part3=123 +POSTHOOK: query: insert into testB partition (part1='US', part2='PQR', part3='123') +values ( '600', '700', 'abc'), ('601', '701', 'abcd') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testb@part1=US/part2=PQR/part3=123 +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=PQR,part3=123).cola SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=PQR,part3=123).colb SCRIPT [] +POSTHOOK: Lineage: testb PARTITION(part1=US,part2=PQR,part3=123).colc SCRIPT [] +PREHOOK: query: create view viewDeterministicUDFA partitioned on (vpart1, vpart2, vpart3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testa +where part1 in ('US', 'CA') +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@testa +PREHOOK: Output: database:default +PREHOOK: Output: default@viewDeterministicUDFA +POSTHOOK: query: create view viewDeterministicUDFA partitioned on (vpart1, vpart2, vpart3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testa +where part1 in ('US', 'CA') +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@testa +POSTHOOK: Output: database:default +POSTHOOK: Output: default@viewDeterministicUDFA +POSTHOOK: Lineage: viewDeterministicUDFA.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFA.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFA.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFA.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFA.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] +PREHOOK: query: create view viewDeterministicUDFB partitioned on (vpart1, vpart2, vpart3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testb +where part1 in ('US', 'CA') +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@testb +PREHOOK: Output: database:default +PREHOOK: Output: default@viewDeterministicUDFB +POSTHOOK: query: create view viewDeterministicUDFB partitioned on (vpart1, vpart2, vpart3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + cast(part1 as char(2)) as vpart1, + cast(part2 as char(3)) as vpart2, + cast(part3 as char(3)) as vpart3 + from testb +where part1 in ('US', 'CA') +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@testb +POSTHOOK: Output: database:default +POSTHOOK: Output: default@viewDeterministicUDFB +POSTHOOK: Lineage: viewDeterministicUDFB.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFB.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] +POSTHOOK: Lineage: viewDeterministicUDFB.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] +PREHOOK: query: create view viewNoUDFA partitioned on (part1, part2, part3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + part1, + part2, + part3 + from testa +where part1 in ('US', 'CA') +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@testa +PREHOOK: Output: database:default +PREHOOK: Output: default@viewNoUDFA +POSTHOOK: query: create view viewNoUDFA partitioned on (part1, part2, part3) as select + cast(col1 as decimal(38,18)) as vcol1, + cast(col2 as decimal(38,18)) as vcol2, + cast(col3 as decimal(38,18)) as vcol3, + cast(col4 as decimal(38,18)) as vcol4, + cast(col5 as char(10)) as vcol5, + part1, + part2, + part3 + from testa +where part1 in ('US', 'CA') +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@testa +POSTHOOK: Output: database:default +POSTHOOK: Output: default@viewNoUDFA +POSTHOOK: Lineage: viewNoUDFA.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFA.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFA.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFA.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFA.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] +PREHOOK: query: create view viewNoUDFB partitioned on (part1, part2, part3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + part1, + part2, + part3 + from testb +where part1 in ('US', 'CA') +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@testb +PREHOOK: Output: database:default +PREHOOK: Output: default@viewNoUDFB +POSTHOOK: query: create view viewNoUDFB partitioned on (part1, part2, part3) as select + cast(cola as decimal(38,18)) as vcolA, + cast(colb as decimal(38,18)) as vcolB, + cast(colc as char(10)) as vcolC, + part1, + part2, + part3 + from testb +where part1 in ('US', 'CA') +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@testb +POSTHOOK: Output: database:default +POSTHOOK: Output: default@viewNoUDFB +POSTHOOK: Lineage: viewNoUDFB.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFB.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] +POSTHOOK: Lineage: viewNoUDFB.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] +PREHOOK: query: explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200' +PREHOOK: type: QUERY +POSTHOOK: query: explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: testa + filterExpr: (part1) IN ('US', 'CA') (type: boolean) + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: CAST( col1 AS decimal(38,18)) (type: decimal(38,18)), CAST( col2 AS decimal(38,18)) (type: decimal(38,18)), CAST( col3 AS decimal(38,18)) (type: decimal(38,18)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 'US' (type: char(2)), 'DEF' (type: char(3)), '200' (type: char(3)) + sort order: +++ + Map-reduce partition columns: 'US' (type: char(2)), 'DEF' (type: char(3)), '200' (type: char(3)) + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)) + TableScan + alias: testb + filterExpr: (part1) IN ('US', 'CA') (type: boolean) + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: CAST( cola AS decimal(38,18)) (type: decimal(38,18)), CAST( colb AS decimal(38,18)) (type: decimal(38,18)), CAST( part1 AS CHAR(2)) (type: char(2)), CAST( part2 AS CHAR(3)) (type: char(3)) + outputColumnNames: _col0, _col1, _col3, _col4 + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: char(2)), _col4 (type: char(3)), '200' (type: char(3)) + sort order: +++ + Map-reduce partition columns: _col3 (type: char(2)), _col4 (type: char(3)), '200' (type: char(3)) + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col5 (type: char(2)), _col6 (type: char(3)), _col7 (type: char(3)) + 1 _col3 (type: char(2)), _col4 (type: char(3)), _col5 (type: char(3)) + outputColumnNames: _col0, _col1, _col2, _col8, _col9 + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col8 (type: decimal(38,18)), _col9 (type: decimal(38,18)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200' +PREHOOK: type: QUERY +PREHOOK: Input: default@testa +PREHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 +PREHOOK: Input: default@testb +PREHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 +PREHOOK: Input: default@viewdeterministicudfa +PREHOOK: Input: default@viewdeterministicudfb +#### A masked pattern was here #### +POSTHOOK: query: select vcol1, vcol2, vcol3, vcola, vcolb +from viewDeterministicUDFA a inner join viewDeterministicUDFB b +on a.vpart1 = b.vpart1 +and a.vpart2 = b.vpart2 +and a.vpart3 = b.vpart3 +and a.vpart1 = 'US' +and a.vpart2 = 'DEF' +and a.vpart3 = '200' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testa +POSTHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 +POSTHOOK: Input: default@testb +POSTHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 +POSTHOOK: Input: default@viewdeterministicudfa +POSTHOOK: Input: default@viewdeterministicudfb +#### A masked pattern was here #### +12.341000000000000000 1001.000000000000000000 2001.000000000000000000 601.000000000000000000 701.000000000000000000 +12.341000000000000000 1001.000000000000000000 2001.000000000000000000 600.000000000000000000 700.000000000000000000 +12.340000000000000000 100.000000000000000000 200.000000000000000000 601.000000000000000000 701.000000000000000000 +12.340000000000000000 100.000000000000000000 200.000000000000000000 600.000000000000000000 700.000000000000000000 +PREHOOK: query: explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200' +PREHOOK: type: QUERY +POSTHOOK: query: explain +select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: testa + filterExpr: ((part1) IN ('US', 'CA') and (part1 = 'US') and (part2 = 'DEF') and (part3 = '200')) (type: boolean) + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: CAST( col1 AS decimal(38,18)) (type: decimal(38,18)), CAST( col2 AS decimal(38,18)) (type: decimal(38,18)), CAST( col3 AS decimal(38,18)) (type: decimal(38,18)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 'US' (type: string), 'DEF' (type: string), '200' (type: string) + sort order: +++ + Map-reduce partition columns: 'US' (type: string), 'DEF' (type: string), '200' (type: string) + Statistics: Num rows: 2 Data size: 4580 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)) + TableScan + alias: testb + filterExpr: ((part1) IN ('US', 'CA') and (part3 = '200') and part1 is not null and part2 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: CAST( cola AS decimal(38,18)) (type: decimal(38,18)), CAST( colb AS decimal(38,18)) (type: decimal(38,18)), part1 (type: string), part2 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: string), '200' (type: string) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: string), '200' (type: string) + Statistics: Num rows: 2 Data size: 3180 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col5 (type: string), _col6 (type: string), _col7 (type: string) + 1 _col3 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col8, _col9 + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col8 (type: decimal(38,18)), _col9 (type: decimal(38,18)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 5038 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200' +PREHOOK: type: QUERY +PREHOOK: Input: default@testa +PREHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 +PREHOOK: Input: default@testb +PREHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 +PREHOOK: Input: default@viewnoudfa +PREHOOK: Input: default@viewnoudfb +#### A masked pattern was here #### +POSTHOOK: query: select vcol1, vcol2, vcol3, vcola, vcolb +from viewNoUDFA a inner join viewNoUDFB b +on a.part1 = b.part1 +and a.part2 = b.part2 +and a.part3 = b.part3 +and a.part1 = 'US' +and a.part2 = 'DEF' +and a.part3 = '200' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testa +POSTHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 +POSTHOOK: Input: default@testb +POSTHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 +POSTHOOK: Input: default@viewnoudfa +POSTHOOK: Input: default@viewnoudfb +#### A masked pattern was here #### +12.341000000000000000 1001.000000000000000000 2001.000000000000000000 601.000000000000000000 701.000000000000000000 +12.341000000000000000 1001.000000000000000000 2001.000000000000000000 600.000000000000000000 700.000000000000000000 +12.340000000000000000 100.000000000000000000 200.000000000000000000 601.000000000000000000 701.000000000000000000 +12.340000000000000000 100.000000000000000000 200.000000000000000000 600.000000000000000000 700.000000000000000000 diff --git ql/src/test/results/clientpositive/ppd_udf_col.q.out ql/src/test/results/clientpositive/ppd_udf_col.q.out index 8d858f50c7c88255e1268916efccb5c2374db5a3..dfc2d0465d49f7c77452113d3791104a05aa42ef 100644 --- ql/src/test/results/clientpositive/ppd_udf_col.q.out +++ ql/src/test/results/clientpositive/ppd_udf_col.q.out @@ -366,3 +366,412 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = 100) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 100) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 <= 0.1) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = 100) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 100) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col2 <= 0.1) and (_col2 > 0.1)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: ((key = 100) and ((value * 10) <= 200.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((value * 10) <= 200.0D) and (key = 100)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double), (value * 10) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = 100) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 100) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 <= 0.1) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM +( +SELECT key, randum123 +FROM (SELECT *, cast(rand() as double) AS randum123 FROM src WHERE key = 100) a +WHERE randum123 <=0.1)s WHERE s.randum123>0.1 LIMIT 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = 100) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 100) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 <= 0.1) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 > 0.1) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 100) and false) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT key,randum123, v10 +FROM (SELECT *, cast(rand() as double) AS randum123, value*10 AS v10 FROM src WHERE key = 100) a +WHERE a.v10 <= 200 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: ((key = 100) and ((value * 10) <= 200.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((value * 10) <= 200.0D) and (key = 100)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), rand() (type: double), (value * 10) (type: double) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col3 <= 200.0D) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: double), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/union_offcbo.q.out ql/src/test/results/clientpositive/union_offcbo.q.out index ab0e394f3f60634dba181a6d0c90f79ef2ad081e..ce27bf29324a384e39daf314f84228bf3b711324 100644 --- ql/src/test/results/clientpositive/union_offcbo.q.out +++ ql/src/test/results/clientpositive/union_offcbo.q.out @@ -288,21 +288,18 @@ STAGE PLANS: outputColumnNames: _col8, _col9, _col10, _col12, _col13, _col16, _col17, _col18, _col19 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((NVL(_col8,-1) <> NVL(_col18,-1)) or (NVL(_col9,-1) <> NVL(_col19,-1))) and _col18 is not null) (type: boolean) + predicate: (((NVL(_col8,-1) <> NVL(_col18,-1)) or (NVL(_col9,-1) <> NVL(_col19,-1))) and (CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and _col18 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: _col10 (type: bigint), _col16 (type: string), _col17 (type: bigint), _col13 (type: string), _col18 (type: string), _col19 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col12 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col18 is not null and _col8 is null and (_col12 <= '2016-02-05'))) THEN ('RET') WHEN (((_col18 = _col8) and (_col19 <> _col9))) THEN ('A_INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -383,21 +380,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col6, _col7, _col8, _col9, _col11, _col18, _col19 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((NVL(_col8,-1) <> NVL(_col18,-1)) or (NVL(_col9,-1) <> NVL(_col19,-1))) and _col8 is not null) (type: boolean) + predicate: (((NVL(_col8,-1) <> NVL(_col18,-1)) or (NVL(_col9,-1) <> NVL(_col19,-1))) and (CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END <> 'RET') and _col8 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col6 (type: string), _col7 (type: bigint), '2099-12-31' (type: string), _col8 (type: string), _col9 (type: string), CASE WHEN ((_col18 is not null and _col8 is null and (_col11 <= _col1))) THEN ('DEL') WHEN (((_col18 is null and _col8 is not null) or ((_col18 = _col8) and (_col19 <> _col9)))) THEN ('INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (_col7 <> 'RET') (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator