diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4393a2825e..645e99a92e 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1663,7 +1663,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_CBO_CNF_NODES_LIMIT("hive.cbo.cnf.maxnodes", -1, "When converting to conjunctive normal form (CNF), fail if" + "the expression exceeds this threshold; the threshold is expressed in terms of number of nodes (leaves and" + "interior nodes). -1 to not set up a threshold."), - HIVE_CBO_RETPATH_HIVEOP("hive.cbo.returnpath.hiveop", false, "Flag to control calcite plan to hive operator conversion"), + HIVE_CBO_RETPATH_HIVEOP("hive.cbo.returnpath.hiveop", "supported", "Flag to control calcite plan to hive operator conversion (true/false/supported)"), HIVE_CBO_EXTENDED_COST_MODEL("hive.cbo.costmodel.extended", false, "Flag to control enabling the extended cost model based on" + "CPU, IO and cardinality. Otherwise, the cost model is based on cardinality."), HIVE_CBO_COST_MODEL_CPU("hive.cbo.costmodel.cpu", "0.000001", "Default cost of a comparison"), diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java index a437998490..3c9e21639f 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java @@ -30,7 +30,7 @@ HiveConf conf = defaultConf(); conf.setBoolVar(HiveConf.ConfVars.LLAP_OUTPUT_FORMAT_ARROW, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_FILESINK_ARROW_NATIVE_ENABLED, true); - conf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP, true); + conf.setVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP, "true"); BaseJdbcWithMiniLlap.beforeTest(conf); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rewrite/AlterMaterializedViewRewriteOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rewrite/AlterMaterializedViewRewriteOperation.java index 426daa6db6..1d331660ee 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rewrite/AlterMaterializedViewRewriteOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rewrite/AlterMaterializedViewRewriteOperation.java @@ -27,8 +27,10 @@ import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.CalcitePlanner; import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; /** * Operation process of enabling/disabling materialized view rewrite. @@ -56,7 +58,9 @@ public int execute() throws HiveException { planner.initCtx(ctx); planner.init(false); - RelNode plan = planner.genLogicalPlan(ParseUtils.parse(newMV.getViewExpandedText())); + ASTNode root = ParseUtils.parse(newMV.getViewExpandedText()); + ReturnPathManager.init(context.getConf(), root); + RelNode plan = planner.genLogicalPlan(root); if (plan == null) { String msg = "Cannot enable automatic rewriting for materialized view."; if (ctx.getCboInfo() != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 7c841ba48e..46358a4ebf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -76,7 +77,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); - final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean returnPathEnabled = ReturnPathManager.shouldUse(); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(cboEnabled && returnPathEnabled && cboSucceeded) { return pctx; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 25e9cd0482..96dec4c54a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.optimizer.stats.annotation.AnnotateWithStatistics; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; @@ -196,7 +197,7 @@ public void initialize(HiveConf hiveConf) { } transformations.add(new NonBlockingOpDeDupProc()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEIDENTITYPROJECTREMOVER) - && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + && !ReturnPathManager.shouldUse()) { transformations.add(new IdentityProjectRemover()); } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVELIMITOPTENABLE)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index 7e6a361dd5..c041b8e7cb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -54,7 +55,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); - final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean returnPathEnabled = ReturnPathManager.shouldUse(); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(!(cboEnabled && returnPathEnabled && cboSucceeded)) { return pctx; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index ea5fa3f4c3..f40d7ca3ff 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -464,7 +464,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // 0. Gen Optimized Plan RelNode newPlan = logicalPlan(); - if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (ReturnPathManager.shouldUse()) { if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) { throw new SemanticException("Create view is not supported in cbo return path."); } @@ -2351,7 +2351,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv } // 9. Run rules to aid in translation from Calcite tree to Hive tree - if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (ReturnPathManager.shouldUse()) { // 9.1. Merge join into multijoin operators (if possible) generatePartialProgram(program, true, HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER, @@ -2901,7 +2901,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 2. if returnpath is on and hivetestmode is on bail if (qb.getParseInfo().getTabSample(tableAlias) != null || getNameToSplitSampleMap().containsKey(tableAlias) - || (conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) && (conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)) ) { + || (ReturnPathManager.shouldUse()) && (conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)) ) { String msg = String.format("Table Sample specified for %s." + " Currently we don't support Table Sample clauses in CBO," + " turn off cbo for queries on tableSamples.", tableAlias); @@ -3025,8 +3025,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc db, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() + getAliasId(tableAlias, qb), ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); @@ -3038,7 +3037,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, getAliasId(tableAlias, qb), - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), + ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); final String dataBaseType = tabMetaData.getProperty(Constants.JDBC_DATABASE_TYPE); @@ -3083,8 +3082,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // Build Hive Table Scan Rel tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() + getAliasId(tableAlias, qb), ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 537af207ae..38c5573ab9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -548,6 +548,7 @@ public static RelNode parseQuery(HiveConf conf, String viewQuery) final Context ctx = new Context(conf); ctx.setIsLoadingMaterializedView(true); final ASTNode ast = parse(viewQuery, ctx); + ReturnPathManager.init(conf, ast); final CalcitePlanner analyzer = getAnalyzer(conf, ctx); return analyzer.genLogicalPlan(ast); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java new file mode 100644 index 0000000000..f293fa622a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * This is a temporary class for the time while we are in transition to use return path only. + * It's purpose is to decide if Hive should use return path. + */ +public final class ReturnPathManager { + private ReturnPathManager() { + throw new UnsupportedOperationException("ReturnPathManager should not be instantiated"); + } + + private static ThreadLocal useReturnPath = new ThreadLocal<>(); + + public static void init(HiveConf conf, ASTNode root) { + boolean use = shouldUseReturnPath(conf, root); + useReturnPath.set(use); + } + + public static boolean shouldUse() { + return useReturnPath.get(); + } + + private static boolean shouldUseReturnPath(HiveConf conf, ASTNode root) { + String confVal = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + if ("true".equals(confVal)) { + return true; + } else if ("false".equals(confVal)) { + return false; + } else if ("supported".contentEquals(confVal)) { + return isReturnPathSupported(root); + } else { + throw new IllegalStateException(String.format("Invalid value for %s: %s", + HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP.varname, confVal )); + } + } + + private static boolean isReturnPathSupported(ASTNode ast) { + // Here is where we may add the logic in the future for supported operations + return false; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 60bfba826d..2e4a6c57e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12425,6 +12425,7 @@ private void removeASTChild(ASTNode node) { @SuppressWarnings("checkstyle:methodlength") void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticException { LOG.info("Starting Semantic Analysis"); + ReturnPathManager.init(conf, ast); // 1. Generate Resolved Parse tree from syntax tree boolean needsTransform = needsTransform(); //change the location of position alias process here