From 9d02cee349c6546ec327756bebed01bdddd93a3d Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 16 Mar 2017 00:23:47 -0700 Subject: [PATCH] HIVE-16230 : Enable CBO in presence of hints --- .../calcite/CalciteSemanticException.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 14 ++-- ql/src/test/queries/clientpositive/comments.q | 4 +- ql/src/test/results/clientpositive/comments.q.out | 76 ++++++++++++---------- 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java index 5b2c9c0..0c6996c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java @@ -31,7 +31,7 @@ public enum UnsupportedFeature { Distinct_without_an_aggreggation, Duplicates_in_RR, Filter_expression_with_non_boolean_return_type, - Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, + Having_clause_without_any_groupby, Invalid_column_reference, Invalid_decimal, Less_than_equal_greater_than, Others, Same_name_in_multiple_expressions, Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery, Table_sample_clauses, UDTF, Union_type, Unique_join diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 34a6dd4..dc3544d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -441,7 +441,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept throw new SemanticException(e); } else if( e instanceof CalciteViewSemanticException) { - // non-cbo path retries to execute create view and + // non-cbo path retries to execute create view and // we believe it will throw the same error message throw new SemanticException(e); } @@ -1008,7 +1008,7 @@ public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) { } return null; } - + public ASTNode depthFirstSearch(ASTNode ast, int token) { searchQueue.clear(); searchQueue.add(ast); @@ -1430,7 +1430,7 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { if (viewScan instanceof DruidQuery) { final DruidQuery dq = (DruidQuery) viewScan; newViewScan = DruidQuery.create(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), - (RelOptHiveTable) viewScan.getTable(), dq.getDruidTable(), + viewScan.getTable(), dq.getDruidTable(), ImmutableList.of(dq.getTableScan())); } else { newViewScan = new HiveTableScan(optCluster, optCluster.traitSetOf(HiveRelNode.CONVENTION), @@ -3484,13 +3484,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel, int posn = 0; boolean hintPresent = (selExprList.getChild(0).getType() == HiveParser.QUERY_HINT); if (hintPresent) { - String hint = ctx.getTokenRewriteStream().toString( - selExprList.getChild(0).getTokenStartIndex(), - selExprList.getChild(0).getTokenStopIndex()); - String msg = String.format("Hint specified for %s." - + " Currently we don't support hints in CBO, turn off cbo to use hints.", hint); - LOG.debug(msg); - throw new CalciteSemanticException(msg, UnsupportedFeature.Hint); + posn++; } // 4. Bailout if select involves Transform diff --git a/ql/src/test/queries/clientpositive/comments.q b/ql/src/test/queries/clientpositive/comments.q index cacb689..b03cd3f 100644 --- a/ql/src/test/queries/clientpositive/comments.q +++ b/ql/src/test/queries/clientpositive/comments.q @@ -15,7 +15,9 @@ select /**/ key /* */ from src limit 1; */ select /* */ key from src limit 1; - +set hive.auto.convert.join=true; select /*+ MAPJOIN(a) */ count(*) from src a join src b on a.key = b.key where a.key > 0; explain extended select /*+ MAPJOIN(a) */ count(*) from src a join src b on a.key = b.key where a.key > 0; + +reset hive.auto.convert.join; diff --git a/ql/src/test/results/clientpositive/comments.q.out b/ql/src/test/results/clientpositive/comments.q.out index 4395cb2..53a766f 100644 --- a/ql/src/test/results/clientpositive/comments.q.out +++ b/ql/src/test/results/clientpositive/comments.q.out @@ -76,34 +76,38 @@ PREHOOK: type: QUERY POSTHOOK: query: explain extended select /*+ MAPJOIN(a) */ count(*) from src a join src b on a.key = b.key where a.key > 0 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-1 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-1 + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-2 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-5 Map Reduce Local Work Alias -> Map Local Tables: - a + $hdt$_0:a Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - a + $hdt$_0:a TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (key > 0) (type: boolean) + predicate: (UDFToDouble(key) > 0.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan @@ -112,28 +116,32 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key > 0) (type: boolean) + predicate: (UDFToDouble(key) > 0.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - Position of Big Table: 1 - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - null sort order: - sort order: + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false Local Work: Map Reduce Local Work Path -> Alias: @@ -186,7 +194,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [b] + /src [$hdt$_1:b] Needs Tagging: false Reduce Operator Tree: Group By Operator -- 2.10.1 (Apple Git-78)