diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 608ec04fe7..195952f69d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -431,6 +431,7 @@ minillap.query.files=acid_bucket_pruning.q,\ global_limit.q,\ dynamic_partition_pruning_2.q,\ results_cache_diff_fs.q,\ + results_cache_with_auth.q,\ tez_union_dynamic_partition.q,\ tez_union_dynamic_partition_2.q,\ unionDistinct_1.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java index cfdc3eb9a7..75ee86de1b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java @@ -28,11 +28,11 @@ import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexOver; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.Util; import org.apache.calcite.util.mapping.Mapping; @@ -82,7 +82,7 @@ public HiveProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, * aliases of the expressions */ public static HiveProject create(RelNode child, List exps, - List fieldNames) throws CalciteSemanticException{ + List fieldNames) throws CalciteSemanticException{ RelOptCluster cluster = child.getCluster(); // 1 Ensure columnNames are unique - CALCITE-411 @@ -201,4 +201,8 @@ public boolean isSynthetic() { return shuttle.visit(this); } + public boolean containsOver() { + return RexOver.containsOver(this.getChildExps(), null); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java index b163052c3b..f0b2469918 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.rules.JoinProjectTransposeRule; @@ -98,4 +99,21 @@ private HiveJoinProjectTransposeRule( super(operand, description, includeOuter, relBuilderFactory); } + public void onMatch(RelOptRuleCall call) { + HiveProject proj; + if (hasLeftChild(call)) { + proj = call.rel(1); + if(proj.containsOver()) { + return; + } + } + if (hasRightChild(call)) { + proj = (HiveProject) getRightChild(call); + if (proj.containsOver()) { + return; + } + } + super.onMatch(call); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 8986315690..f8bf50b08b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -375,8 +375,7 @@ public RelNode genLogicalPlan(ASTNode ast) throws SemanticException { PreCboCtx cboCtx = new PreCboCtx(); //change the location of position alias process here processPositionAlias(ast); - this.setAST(ast); - if (!genResolvedParseTree(cboCtx)) { + if (!genResolvedParseTree(ast, cboCtx)) { return null; } ASTNode queryForCbo = ast; @@ -428,13 +427,10 @@ private static RelOptPlanner createPlanner( @Override @SuppressWarnings("rawtypes") - Operator genOPTree(PlannerContext plannerCtx) throws SemanticException { + Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { Operator sinkOp = null; boolean skipCalcitePlan = false; - // Save original AST in case CBO tampers with the contents of ast to guarantee fail-safe behavior. - final ASTNode originalAst = (ASTNode) ParseDriver.adaptor.dupTree(this.getAST()); - if (!runCBO) { skipCalcitePlan = true; } else { @@ -449,14 +445,14 @@ Operator genOPTree(PlannerContext plannerCtx) throws SemanticException { // SA. We rely on the fact that CBO ignores the unknown tokens (create // table, destination), so if the query is otherwise ok, it is as if we // did remove those and gave CBO the proper AST. That is kinda hacky. - ASTNode queryForCbo = this.getAST(); + ASTNode queryForCbo = ast; if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) { queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } Pair canCBOHandleReason = canCBOHandleAst(queryForCbo, getQB(), cboCtx); runCBO = canCBOHandleReason.left; if (queryProperties.hasMultiDestQuery()) { - handleMultiDestQuery(this.getAST(), cboCtx); + handleMultiDestQuery(ast, cboCtx); } if (runCBO) { @@ -487,7 +483,7 @@ Operator genOPTree(PlannerContext plannerCtx) throws SemanticException { ASTNode newAST = getOptimizedAST(newPlan); // 1.1. Fix up the query for insert/ctas/materialized views - newAST = fixUpAfterCbo(this.getAST(), newAST, cboCtx); + newAST = fixUpAfterCbo(ast, newAST, cboCtx); // 1.2. Fix up the query for materialization rebuild if (mvRebuildMode == MaterializationRebuildMode.AGGREGATE_REBUILD) { @@ -565,13 +561,13 @@ Operator genOPTree(PlannerContext plannerCtx) throws SemanticException { } } } catch (Exception e) { - LOG.error("CBO failed, skipping CBO. ", e); boolean isMissingStats = noColsMissingStats.get() > 0; if (isMissingStats) { LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO"); this.ctx .setCboInfo("Plan not optimized by CBO due to missing statistics. Please check log for more details."); } else { + LOG.error("CBO failed, skipping CBO. ", e); if (e instanceof CalciteSemanticException) { CalciteSemanticException calciteSemanticException = (CalciteSemanticException) e; UnsupportedFeature unsupportedFeature = calciteSemanticException @@ -617,14 +613,12 @@ else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats runCBO = false; disableJoinMerge = defaultJoinMerge; disableSemJoinReordering = false; - // Make sure originalAst is used from here on. if (reAnalyzeAST) { init(true); prunedPartitions.clear(); // Assumption: At this point Parse Tree gen & resolution will always // be true (since we started out that way). - this.setAST(originalAst); - super.genResolvedParseTree(new PlannerContext()); + super.genResolvedParseTree(ast, new PlannerContext()); skipCalcitePlan = true; } } @@ -641,7 +635,7 @@ else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats } if (skipCalcitePlan) { - sinkOp = super.genOPTree(); + sinkOp = super.genOPTree(ast, plannerCtx); } return sinkOp; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c2514eedb1..93731c5d48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12295,8 +12295,9 @@ private ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, To } } - boolean genResolvedParseTree(PlannerContext plannerCtx) throws SemanticException { + boolean genResolvedParseTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { ASTNode child = ast; + this.ast = ast; viewsExpanded = new ArrayList(); ctesExpanded = new ArrayList(); @@ -12399,12 +12400,7 @@ private void getHintsFromQB(QBExpr qbExpr, List hints) { } } - Operator genOPTree(PlannerContext plannerCtx) throws SemanticException { - // Parameters are not utilized when CBO is disabled. - return genOPTree(); - } - - Operator genOPTree() throws SemanticException { + Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { // fetch all the hints in qb List hintsList = new ArrayList<>(); getHintsFromQB(qb, hintsList); @@ -12458,15 +12454,14 @@ private void removeASTChild(ASTNode node) { } @SuppressWarnings("checkstyle:methodlength") - void analyzeInternal(final ASTNode astToAnalyze, Supplier pcf) throws SemanticException { + void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticException { LOG.info("Starting Semantic Analysis"); + // 1. Generate Resolved Parse tree from syntax tree boolean needsTransform = needsTransform(); //change the location of position alias process here - processPositionAlias(astToAnalyze); + processPositionAlias(ast); PlannerContext plannerCtx = pcf.get(); - this.setAST(astToAnalyze); - // 1. Generate Resolved Parse tree from syntax tree - if (!genResolvedParseTree(plannerCtx)) { + if (!genResolvedParseTree(ast, plannerCtx)) { return; } @@ -12494,7 +12489,7 @@ void analyzeInternal(final ASTNode astToAnalyze, Supplier pcf) t boolean isCacheEnabled = isResultsCacheEnabled(); QueryResultsCache.LookupInfo lookupInfo = null; if (isCacheEnabled && !needsTransform && queryTypeCanUseCache()) { - lookupInfo = createLookupInfoForQuery(astToAnalyze); + lookupInfo = createLookupInfoForQuery(ast); if (checkResultsCache(lookupInfo, false)) { return; } @@ -12506,13 +12501,13 @@ void analyzeInternal(final ASTNode astToAnalyze, Supplier pcf) t // If we use CBO and we may apply masking/filtering policies, we create a copy of the ast. // The reason is that the generation of the operator tree may modify the initial ast, // but if we need to parse for a second time, we would like to parse the unmodified ast. - astForMasking = (ASTNode) ParseDriver.adaptor.dupTree(astToAnalyze); + astForMasking = (ASTNode) ParseDriver.adaptor.dupTree(ast); } else { - astForMasking = astToAnalyze; + astForMasking = ast; } // 2. Gen OP Tree from resolved Parse Tree - Operator sinkOp = genOPTree(plannerCtx); + Operator sinkOp = genOPTree(ast, plannerCtx); boolean usesMasking = false; if (!unparseTranslator.isEnabled() && @@ -12527,12 +12522,11 @@ void analyzeInternal(final ASTNode astToAnalyze, Supplier pcf) t init(true); //change the location of position alias process here processPositionAlias(rewrittenAST); - this.setAST(rewrittenAST); - genResolvedParseTree(plannerCtx); + genResolvedParseTree(rewrittenAST, plannerCtx); if (this instanceof CalcitePlanner) { ((CalcitePlanner) this).resetCalciteConfiguration(); } - sinkOp = genOPTree(plannerCtx); + sinkOp = genOPTree(rewrittenAST, plannerCtx); } } @@ -12540,7 +12534,7 @@ void analyzeInternal(final ASTNode astToAnalyze, Supplier pcf) t // In the case that row or column masking/filtering was required, we do not support caching. // TODO: Enable caching for queries with masking/filtering if (isCacheEnabled && needsTransform && !usesMasking && queryTypeCanUseCache()) { - lookupInfo = createLookupInfoForQuery(astToAnalyze); + lookupInfo = createLookupInfoForQuery(ast); if (checkResultsCache(lookupInfo, false)) { return; } diff --git a/ql/src/test/queries/clientpositive/results_cache_with_auth.q b/ql/src/test/queries/clientpositive/results_cache_with_auth.q index e08ddcdefa..7ac3ddc9de 100644 --- a/ql/src/test/queries/clientpositive/results_cache_with_auth.q +++ b/ql/src/test/queries/clientpositive/results_cache_with_auth.q @@ -1,5 +1,4 @@ --! qt:authorizer - -- Setup results cache set hive.compute.query.using.stats=false; set hive.query.results.cache.enabled=true; diff --git a/ql/src/test/results/clientpositive/results_cache_with_auth.q.out b/ql/src/test/results/clientpositive/llap/results_cache_with_auth.q.out similarity index 62% rename from ql/src/test/results/clientpositive/results_cache_with_auth.q.out rename to ql/src/test/results/clientpositive/llap/results_cache_with_auth.q.out index e283d51f5e..51c0bca40b 100644 --- a/ql/src/test/results/clientpositive/results_cache_with_auth.q.out +++ b/ql/src/test/results/clientpositive/llap/results_cache_with_auth.q.out @@ -19,50 +19,59 @@ PREHOOK: query: explain select count(*) from results_cache_with_auth_t1 PREHOOK: type: QUERY PREHOOK: Input: default@results_cache_with_auth_t1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select count(*) from results_cache_with_auth_t1 POSTHOOK: type: QUERY POSTHOOK: Input: default@results_cache_with_auth_t1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: results_cache_with_auth_t1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: results_cache_with_auth_t1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash + aggregations: count(VALUE._col0) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator