diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 2902d92..3ce1957 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -571,6 +571,7 @@ minillaplocal.query.files=acid_globallimit.q,\ subquery_notin.q,\ subquery_nested_subquery.q, \ subquery_shared_alias.q, \ + subquery_null_agg.q,\ table_access_keys_stats.q,\ tez_bmj_schema_evolution.q,\ tez_dml.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java index 21ddae6..cfac6c0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java @@ -29,13 +29,13 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.parse.SubQueryDiagnostic.QBSubQueryRewrite; import org.apache.hadoop.hive.ql.parse.SubQueryUtils.ISubQueryJoinInfo; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory.DefaultExprProcessor; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.ql.parse.SubQueryDiagnostic.QBSubQueryRewrite; public class QBSubQuery implements ISubQueryJoinInfo { @@ -327,14 +327,13 @@ protected ColumnInfo resolveDot(ASTNode node) { try { TypeCheckCtx tcCtx = new TypeCheckCtx(parentQueryRR); String str = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(1).getText()); - ExprNodeDesc idDesc = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, - str.toLowerCase()); - ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) - defaultExprProcessor.process(node, stack, tcCtx, (Object) null, idDesc); - if ( colDesc != null ) { - String[] qualName = parentQueryRR.reverseLookup(colDesc.getColumn()); - return parentQueryRR.get(qualName[0], qualName[1]); - } + ExprNodeDesc idDesc = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, str.toLowerCase()); + Object desc = defaultExprProcessor.process(node, stack, tcCtx, (Object) null, idDesc); + if (desc != null && desc instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) desc; + String[] qualName = parentQueryRR.reverseLookup(colDesc.getColumn()); + return parentQueryRR.get(qualName[0], qualName[1]); + } } catch(SemanticException se) { } return null; diff --git ql/src/test/queries/clientpositive/subquery_null_agg.q ql/src/test/queries/clientpositive/subquery_null_agg.q new file mode 100644 index 0000000..ca0d5c7 --- /dev/null +++ ql/src/test/queries/clientpositive/subquery_null_agg.q @@ -0,0 +1,22 @@ +set hive.strict.checks.cartesian.product=false; + +CREATE TABLE table_7 (int_col INT); + +explain +SELECT +(t1.int_col) * (t1.int_col) AS int_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) t1 +WHERE +(False) NOT IN (SELECT +False AS boolean_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) tt1 +WHERE +(t1.int_col) = (tt1.int_col)); diff --git ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out new file mode 100644 index 0000000..852d919 --- /dev/null +++ ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out @@ -0,0 +1,200 @@ +PREHOOK: query: CREATE TABLE table_7 (int_col INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table_7 +POSTHOOK: query: CREATE TABLE table_7 (int_col INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_7 +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +PREHOOK: query: explain +SELECT +(t1.int_col) * (t1.int_col) AS int_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) t1 +WHERE +(False) NOT IN (SELECT +False AS boolean_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) tt1 +WHERE +(t1.int_col) = (tt1.int_col)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT +(t1.int_col) * (t1.int_col) AS int_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) t1 +WHERE +(False) NOT IN (SELECT +False AS boolean_col +FROM ( +SELECT +MIN(NULL) OVER () AS int_col +FROM table_7 +) tt1 +WHERE +(t1.int_col) = (tt1.int_col)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count(), count(false) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: false (type: boolean), true (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: boolean), _col1 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 + 1 + outputColumnNames: _col1, _col4 + Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col4 is null or (_col1 = 0)) (type: boolean) + Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: null (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col1 >= _col0) or (_col0 = 0)) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: boolean), KEY._col1 (type: boolean) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: _col1 (type: boolean) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col1 (type: boolean) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +