diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bf03fe6ea3..1ea714fac5 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1663,7 +1663,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_CBO_CNF_NODES_LIMIT("hive.cbo.cnf.maxnodes", -1, "When converting to conjunctive normal form (CNF), fail if" + "the expression exceeds this threshold; the threshold is expressed in terms of number of nodes (leaves and" + "interior nodes). -1 to not set up a threshold."), - HIVE_CBO_RETPATH_HIVEOP("hive.cbo.returnpath.hiveop", false, "Flag to control calcite plan to hive operator conversion"), + HIVE_CBO_RETPATH_HIVEOP("hive.cbo.returnpath.hiveop", "supported", "Flag to control calcite plan to hive operator conversion (on/off/supported)"), HIVE_CBO_EXTENDED_COST_MODEL("hive.cbo.costmodel.extended", false, "Flag to control enabling the extended cost model based on" + "CPU, IO and cardinality. Otherwise, the cost model is based on cardinality."), HIVE_CBO_COST_MODEL_CPU("hive.cbo.costmodel.cpu", "0.000001", "Default cost of a comparison"), diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatWithoutReturnPath.java similarity index 89% rename from itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java rename to itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatWithoutReturnPath.java index a437998490..6c6ae9787c 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatReturnPath.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNewGetSplitsFormatWithoutReturnPath.java @@ -24,13 +24,13 @@ /** * TestNewGetSplitsFormatReturnPath. */ -public class TestNewGetSplitsFormatReturnPath extends TestNewGetSplitsFormat { +public class TestNewGetSplitsFormatWithoutReturnPath extends TestNewGetSplitsFormat { @BeforeClass public static void beforeTest() throws Exception { HiveConf conf = defaultConf(); conf.setBoolVar(HiveConf.ConfVars.LLAP_OUTPUT_FORMAT_ARROW, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_FILESINK_ARROW_NATIVE_ENABLED, true); - conf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP, true); + conf.setVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP, "off"); BaseJdbcWithMiniLlap.beforeTest(conf); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 7c841ba48e..46358a4ebf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -76,7 +77,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); - final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean returnPathEnabled = ReturnPathManager.shouldUse(); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(cboEnabled && returnPathEnabled && cboSucceeded) { return pctx; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 25e9cd0482..96dec4c54a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.optimizer.stats.annotation.AnnotateWithStatistics; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; @@ -196,7 +197,7 @@ public void initialize(HiveConf hiveConf) { } transformations.add(new NonBlockingOpDeDupProc()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEIDENTITYPROJECTREMOVER) - && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + && !ReturnPathManager.shouldUse()) { transformations.add(new IdentityProjectRemover()); } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVELIMITOPTENABLE)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index 7e6a361dd5..c041b8e7cb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ReturnPathManager; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -54,7 +55,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); - final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + final boolean returnPathEnabled = ReturnPathManager.shouldUse(); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(!(cboEnabled && returnPathEnabled && cboSucceeded)) { return pctx; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 2b9caac03f..785afff30d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -464,7 +464,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // 0. Gen Optimized Plan RelNode newPlan = logicalPlan(); - if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (ReturnPathManager.shouldUse()) { if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) { throw new SemanticException("Create view is not supported in cbo return path."); } @@ -2351,7 +2351,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv } // 9. Run rules to aid in translation from Calcite tree to Hive tree - if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (ReturnPathManager.shouldUse()) { // 9.1. Merge join into multijoin operators (if possible) generatePartialProgram(program, true, HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER, @@ -2901,7 +2901,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 2. if returnpath is on and hivetestmode is on bail if (qb.getParseInfo().getTabSample(tableAlias) != null || getNameToSplitSampleMap().containsKey(tableAlias) - || (conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) && (conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)) ) { + || (ReturnPathManager.shouldUse()) && (conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)) ) { String msg = String.format("Table Sample specified for %s." + " Currently we don't support Table Sample clauses in CBO," + " turn off cbo for queries on tableSamples.", tableAlias); @@ -3025,8 +3025,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc db, tabNameToTabObject, partitionCache, colStatsCache, noColsMissingStats); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() + getAliasId(tableAlias, qb), ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); @@ -3038,7 +3037,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, getAliasId(tableAlias, qb), - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), + ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); final String dataBaseType = tabMetaData.getProperty(Constants.JDBC_DATABASE_TYPE); @@ -3083,8 +3082,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // Build Hive Table Scan Rel tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, - getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP), qb.isInsideView() + getAliasId(tableAlias, qb), ReturnPathManager.shouldUse(), qb.isInsideView() || qb.getAliasInsideView().contains(tableAlias.toLowerCase())); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java new file mode 100644 index 0000000000..0e4da3beee --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ReturnPathManager.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * This is a temporary class for the time while we are in transition to use return path only. + * It's purpose is to decide if Hive should use return path. + */ +public final class ReturnPathManager { + private ReturnPathManager() { + throw new UnsupportedOperationException("ReturnPathManager should not be instantiated"); + } + + private static ThreadLocal useReturnPath = new ThreadLocal<>(); + + public static void init(HiveConf conf, ASTNode root) { + boolean use = shouldUseReturnPath(conf, root); + useReturnPath.set(use); + } + + public static boolean shouldUse() { + return useReturnPath.get(); + } + + private static boolean shouldUseReturnPath(HiveConf conf, ASTNode root) { + String confVal = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); + if ("on".equals(confVal)) { + return true; + } else if ("off".equals(confVal)) { + return false; + } else if ("supported".contentEquals(confVal)) { + return isReturnPathSupported(root); + } else { + throw new IllegalStateException(String.format("Invalid value for %s: %s", + HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP.varname, confVal )); + } + } + + private static boolean isReturnPathSupported(ASTNode ast) { + return ast.getType() == HiveParser.TOK_QUERY; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ff60ba54b3..a57465df47 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12425,6 +12425,7 @@ private void removeASTChild(ASTNode node) { @SuppressWarnings("checkstyle:methodlength") void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticException { LOG.info("Starting Semantic Analysis"); + ReturnPathManager.init(conf, ast); // 1. Generate Resolved Parse tree from syntax tree boolean needsTransform = needsTransform(); //change the location of position alias process here