diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java index 25227a35d8..efabe5db6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java @@ -119,7 +119,7 @@ public void onMatch(RelOptRuleCall call) { case IS_NULL: conditions.put(conjCall.operands.get(0).toString(), relBuilder.getRexBuilder().makeNullLiteral( - conjCall.operands.get(0).getType().getSqlTypeName())); + conjCall.operands.get(0).getType())); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index 7328b72bba..9e913afcdc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -243,6 +243,7 @@ public static ASTNode literal(RexLiteral literal) { case INTERVAL_SECOND: case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: + case ROW: if (literal.getValue() == null) { return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index edb31992cb..039d9e52ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -160,7 +160,7 @@ public static RelDataType convert(TypeInfo type, RelDataTypeFactory dtFactory) convertedType = convert((UnionTypeInfo) type, dtFactory); break; } - return convertedType; + return dtFactory.createTypeWithNullability(convertedType, true); } public static RelDataType convert(PrimitiveTypeInfo type, RelDataTypeFactory dtFactory) { diff --git a/ql/src/test/queries/clientpositive/structin.q b/ql/src/test/queries/clientpositive/structin.q index 962119574f..32411da805 100644 --- a/ql/src/test/queries/clientpositive/structin.q +++ b/ql/src/test/queries/clientpositive/structin.q @@ -21,3 +21,14 @@ IN ( struct('1234-1111-0074578664','3'), struct('1234-1111-0074578695',1) ); + +CREATE TABLE test_struct +( + f1 string, + demo_struct struct, + datestr string +); + +explain select * from test_struct where datestr='2019-01-01' and demo_struct is not null; + +DROP TABLE test_struct; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index ea8fe5ea96..0d867ba220 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -4008,13 +4008,13 @@ STAGE PLANS: Select Operator expressions: p_name (type: string), p_type (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) null sort order: z sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs @@ -4067,10 +4067,10 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col3 Statistics: Num rows: 28 Data size: 3500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col4 (type: int) + expressions: _col0 (type: string), _col3 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 28 Data size: 3500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out index b0b8169a91..6a8ed39e29 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out @@ -44,13 +44,13 @@ CBO PLAN: HiveSortLimit(sort0=[$1], sort1=[$0], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveProject(ss_customer_sk=[$0], $f1=[$1]) HiveAggregate(group=[{0}], agg#0=[sum($1)]) - HiveProject(ss_customer_sk=[$1], act_sales=[CASE(IS NOT NULL($8), *(CAST(-($3, $8)):DECIMAL(10, 0), $4), *(CAST($3):DECIMAL(10, 0), $4))]) - HiveJoin(condition=[AND(=($5, $0), =($7, $2))], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(ss_item_sk=[$2], ss_customer_sk=[$3], ss_ticket_number=[$9], ss_quantity=[$10], ss_sales_price=[$13]) + HiveProject(ss_customer_sk=[$1], act_sales=[CASE($10, *(CAST(-($3, $9)):DECIMAL(10, 0), $4), $5)]) + HiveJoin(condition=[AND(=($6, $0), =($8, $2))], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(ss_item_sk=[$2], ss_customer_sk=[$3], ss_ticket_number=[$9], ss_quantity=[$10], ss_sales_price=[$13], *=[*(CAST($10):DECIMAL(10, 0), $13)]) HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($9))]) HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) - HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available]) - HiveProject(sr_item_sk=[$2], sr_reason_sk=[$8], sr_ticket_number=[$9], sr_return_quantity=[$10]) + HiveJoin(condition=[=($1, $5)], joinType=[inner], algorithm=[none], cost=[not available]) + HiveProject(sr_item_sk=[$2], sr_reason_sk=[$8], sr_ticket_number=[$9], sr_return_quantity=[$10], IS NOT NULL=[IS NOT NULL($10)]) HiveFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($2), IS NOT NULL($9))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveProject(r_reason_sk=[$0]) diff --git a/ql/src/test/results/clientpositive/perf/tez/query93.q.out b/ql/src/test/results/clientpositive/perf/tez/query93.q.out index d0bde224fb..6a47c90338 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query93.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query93.q.out @@ -69,15 +69,15 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_17] (rows=569016747 width=115) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_15] (rows=569016747 width=118) + Select Operator [SEL_15] (rows=569016747 width=234) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_65] (rows=569016747 width=118) - Conds:RS_12._col0, _col2=RS_74._col0, _col2(Inner),Output:["_col3","_col6","_col8","_col9"] + Merge Join Operator [MERGEJOIN_65] (rows=569016747 width=234) + Conds:RS_12._col0, _col2=RS_74._col0, _col2(Inner),Output:["_col3","_col4","_col7","_col9","_col10","_col11"] <-Map 7 [SIMPLE_EDGE] vectorized SHUFFLE [RS_74] PartitionCols:_col0, _col2 - Select Operator [SEL_73] (rows=575995635 width=122) - Output:["_col0","_col1","_col2","_col3","_col4"] + Select Operator [SEL_73] (rows=575995635 width=234) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] Filter Operator [FIL_72] (rows=575995635 width=122) predicate:(ss_item_sk is not null and ss_ticket_number is not null) TableScan [TS_6] (rows=575995635 width=122) @@ -85,13 +85,13 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0, _col2 - Merge Join Operator [MERGEJOIN_64] (rows=55574563 width=11) - Conds:RS_68._col1=RS_71._col0(Inner),Output:["_col0","_col2","_col3"] + Merge Join Operator [MERGEJOIN_64] (rows=55574563 width=15) + Conds:RS_68._col1=RS_71._col0(Inner),Output:["_col0","_col2","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_68] PartitionCols:_col1 - Select Operator [SEL_67] (rows=55574563 width=15) - Output:["_col0","_col1","_col2","_col3"] + Select Operator [SEL_67] (rows=55574563 width=19) + Output:["_col0","_col1","_col2","_col3","_col4"] Filter Operator [FIL_66] (rows=55574563 width=15) predicate:(sr_reason_sk is not null and sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_0] (rows=57591150 width=15) diff --git a/ql/src/test/results/clientpositive/structin.q.out b/ql/src/test/results/clientpositive/structin.q.out index 0fe88f5125..9267aaddf1 100644 --- a/ql/src/test/results/clientpositive/structin.q.out +++ b/ql/src/test/results/clientpositive/structin.q.out @@ -119,3 +119,71 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: CREATE TABLE test_struct +( + f1 string, + demo_struct struct, + datestr string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_struct +POSTHOOK: query: CREATE TABLE test_struct +( + f1 string, + demo_struct struct, + datestr string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_struct +PREHOOK: query: explain select * from test_struct where datestr='2019-01-01' and demo_struct is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@test_struct +#### A masked pattern was here #### +POSTHOOK: query: explain select * from test_struct where datestr='2019-01-01' and demo_struct is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_struct +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_struct + filterExpr: ((datestr = '2019-01-01') and demo_struct is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 886 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((datestr = '2019-01-01') and demo_struct is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 886 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: f1 (type: string), demo_struct (type: struct), '2019-01-01' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 886 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 886 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP TABLE test_struct +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test_struct +PREHOOK: Output: default@test_struct +POSTHOOK: query: DROP TABLE test_struct +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test_struct +POSTHOOK: Output: default@test_struct