diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java index dee7d7e..3097385 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java @@ -112,7 +112,7 @@ private static void doSemanticAnalysis(SemanticAnalyzer sem, ((SemanticAnalyzer) sem).initParseCtx(subPCtx); LOG.info("Starting Sub-query Semantic Analysis"); - sem.doPhase1(child, qb, sem.initPhase1Ctx()); + sem.doPhase1(child, qb, sem.initPhase1Ctx(), null); LOG.info("Completed phase 1 of Sub-query Semantic Analysis"); sem.getMetaData(qb); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 37cbf7f..becbc38 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -79,7 +79,7 @@ * */ public abstract class BaseSemanticAnalyzer { - private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName()); + protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName()); protected final Hive db; protected final HiveConf conf; protected List> rootTasks; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d8c50e3..7d59fcd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -368,7 +369,7 @@ //flag for partial scan during analyze ... compute statistics protected boolean partialscan; - private volatile boolean runCBO = true; + private volatile boolean runCBO = true; // TODO: why is this volatile? private volatile boolean disableJoinMerge = false; /* @@ -380,6 +381,9 @@ */ private ArrayList ctesExpanded; + /** Not thread-safe. */ + private ASTSearcher astSearcher = new ASTSearcher(); + private static class Phase1Ctx { String dest; int nextNum; @@ -512,7 +516,7 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) case HiveParser.TOK_QUERY: { QB qb = new QB(id, alias, true); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(ast, qb, ctx_1); + doPhase1(ast, qb, ctx_1, null); qbexpr.setOpcode(QBExpr.Opcode.NULLOP); qbexpr.setQB(qb); @@ -1177,6 +1181,30 @@ private String processLateralView(QB qb, ASTNode lateralView) return alias; } + /** The context that doPhase1 uses to populate information pertaining + * to CBO (currently, this is used for CTAS and insert-as-select). */ + private static class PreCboCtx { + enum Type { + NONE, + INSERT, + CTAS, + + UNEXPECTED + } + public ASTNode nodeOfInterest; + public Type type = Type.NONE; + public void set(Type type, ASTNode ast) { + if (this.type != Type.NONE) { + STATIC_LOG.warn("Setting " + type + " when already " + this.type + + "; node " + ast.dump() + " vs old node " + nodeOfInterest.dump()); + this.type = Type.UNEXPECTED; + return; + } + this.type = type; + this.nodeOfInterest = ast; + } + } + /** * Phase 1: (including, but not limited to): * @@ -1194,7 +1222,7 @@ private String processLateralView(QB qb, ASTNode lateralView) * @throws SemanticException */ @SuppressWarnings({"fallthrough", "nls"}) - public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) + public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PreCboCtx cboCtx) throws SemanticException { boolean phase1Result = true; @@ -1238,25 +1266,32 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) String currentDatabase = SessionState.get().getCurrentDatabase(); String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase); qbp.addInsertIntoTable(tab_name); - // TODO: is this supposed to fall thru? case HiveParser.TOK_DESTINATION: ctx_1.dest = "insclause-" + ctx_1.nextNum; ctx_1.nextNum++; + boolean isTmpFileDest = false; + if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) { + ASTNode ch = (ASTNode)ast.getChild(0); + if (ch.getToken().getType() == HiveParser.TOK_DIR + && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) { + ch = (ASTNode)ch.getChild(0); + isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE; + } + } // is there a insert in the subquery - if (qbp.getIsSubQ()) { - ASTNode ch = (ASTNode) ast.getChild(0); - if ((ch.getToken().getType() != HiveParser.TOK_DIR) - || (((ASTNode) ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) { - throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY - .getMsg(ast)); - } + if (qbp.getIsSubQ() && !isTmpFileDest) { + throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast)); + } + if (cboCtx != null && !isTmpFileDest) { + cboCtx.set(PreCboCtx.Type.INSERT, ast); } qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0)); - if (qbp.getClauseNamesForDest().size() > 1) + if (qbp.getClauseNamesForDest().size() > 1) { queryProperties.setMultiDestQuery(true); + } break; case HiveParser.TOK_FROM: @@ -1479,12 +1514,18 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) int child_count = ast.getChildCount(); for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) { // Recurse - phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1); + phase1Result = phase1Result && doPhase1( + (ASTNode)ast.getChild(child_pos), qb, ctx_1, cboCtx); } } return phase1Result; } + private void traceLogAst(ASTNode ast, String what) { + if (!LOG.isTraceEnabled()) return; + LOG.trace(what + ast.dump()); + } + private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { @@ -1759,6 +1800,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } } else { + // This is the only place where isQuery is set to true; it defaults to false. qb.setIsQuery(true); fname = ctx.getMRTmpPath().toString(); ctx.setResDir(new Path(fname)); @@ -2463,7 +2505,7 @@ private Operator genPlanForSubQueryPredicate( ISubQueryJoinInfo subQueryPredicate) throws SemanticException { qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery()); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1); + doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null); getMetaData(qbSQ); Operator op = genPlan(qbSQ); return op; @@ -6188,8 +6230,7 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) ArrayList colInfos = inputRR.getColumnInfos(); // CTAS case: the file output format and serde are defined by the create - // table command - // rather than taking the default value + // table command rather than taking the default value List field_schemas = null; CreateTableDesc tblDesc = qb.getTableDesc(); if (tblDesc != null) { @@ -6210,7 +6251,8 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) if (!("".equals(nm[0])) && nm[1] != null) { colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove `` } - col.setName(colName);; + String ctasColName = fixCtasColumnName(colName, colInfo, inputRR); + col.setName(ctasColName); col.setType(colInfo.getType().getTypeName()); field_schemas.add(col); } @@ -6388,6 +6430,14 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) return output; } + private static String fixCtasColumnName(String colName, ColumnInfo colInfo, RowResolver rr) { + int lastDot = colName.lastIndexOf('.'); + if (lastDot < 0) return colName; // alias is not fully qualified + String nqColumnName = colName.substring(lastDot + 1); + STATIC_LOG.debug("Replacing " + colName + " (produced by CBO) by " + nqColumnName); + return nqColumnName; + } + // Check constraints on acid tables. This includes // * no insert overwrites // * no use of vectorization @@ -9866,11 +9916,14 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // analyze and process the position alias processPositionAlias(ast); + // Check configuration for CBO first. + runCBO = runCBO && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED); // analyze create table command + PreCboCtx cboCtx = runCBO ? new PreCboCtx() : null; if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) { // if it is not CTAS, we don't need to go further and just return - if ((child = analyzeCreateTable(ast, qb)) == null) { + if ((child = analyzeCreateTable(ast, qb, cboCtx)) == null) { return; } } else { @@ -9879,7 +9932,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // analyze create view command if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || - (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { + (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW + && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { child = analyzeCreateView(ast, qb); SessionState.get().setCommandType(HiveOperation.CREATEVIEW); if (child == null) { @@ -9892,7 +9946,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // continue analyzing from the child ASTNode. Phase1Ctx ctx_1 = initPhase1Ctx(); - if (!doPhase1(child, qb, ctx_1)) { + if (!doPhase1(child, qb, ctx_1, cboCtx)) { // if phase1Result false return return; } @@ -9902,20 +9956,15 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { getMetaData(qb); LOG.info("Completed getting MetaData in Semantic Analysis"); - - if (runCBO) { - boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY - || ast.getToken().getType() == HiveParser.TOK_EXPLAIN; - if (!tokenTypeIsQuery || createVwDesc != null - || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) - || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) { - runCBO = false; - } - - if (runCBO) { - disableJoinMerge = true; - } + // Note: for now, we don't actually pass the queryForCbo to CBO, because it accepts qb, not + // AST, and can also access all the private stuff in SA. We rely on the fact that CBO + // ignores the unknown tokens (create table, destination), so if the query is otherwise ok, + // it is as if we did remove those and gave CBO the proper AST. That is kinda hacky. + ASTNode queryForCbo = ast; + if (cboCtx.type == PreCboCtx.Type.CTAS) { + queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } + runCBO = runCBO && canHandleAstForCbo(queryForCbo, qb, cboCtx); // Save the result schema derived from the sink operator produced // by genPlan. This has the correct column names, which clients @@ -9924,6 +9973,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { Operator sinkOp = null; if (runCBO) { + disableJoinMerge = true; OptiqBasedPlanner optiqPlanner = new OptiqBasedPlanner(); boolean reAnalyzeAST = false; @@ -9931,10 +9981,17 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // 1. Gen Optimized AST ASTNode newAST = optiqPlanner.getOptimizedAST(prunedPartitions); + // 1.1. Fix up the query for insert/ctas + newAST = fixUpCtasAndInsertAfterCbo(ast, newAST, cboCtx); + // 2. Regen OP plan from optimized AST init(false); + if (cboCtx.type == PreCboCtx.Type.CTAS) { + // Redo create-table analysis, because it's not part of doPhase1. + newAST = reAnalyzeCtasAfterCbo(newAST); + } ctx_1 = initPhase1Ctx(); - if (!doPhase1(newAST, qb, ctx_1)) { + if (!doPhase1(newAST, qb, ctx_1, null)) { throw new RuntimeException( "Couldn't do phase1 on CBO optimized query plan"); } @@ -10080,6 +10137,118 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { return; } + private ASTNode fixUpCtasAndInsertAfterCbo( + ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) throws SemanticException { + switch (cboCtx.type) { + case NONE: return newAst; // nothing to do + case CTAS: { + // Patch the optimized query back into original CTAS AST, replacing the original query. + replaceASTChild(cboCtx.nodeOfInterest, newAst); + return originalAst; + } + case INSERT: { + // We need to patch the dest back to original into new query. + // This makes assumptions about the structure of the AST. + ASTNode newDest = astSearcher.simpleBreadthFirstSearch( + newAst, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION); + if (newDest == null) { + LOG.error("Cannot find destination after CBO; new ast is "+ newAst.dump()); + throw new SemanticException("Cannot find destination after CBO"); + } + replaceASTChild(newDest, cboCtx.nodeOfInterest); + return newAst; + } + default: throw new AssertionError("Unexpected type " + cboCtx.type); + } + } + + private ASTNode reAnalyzeCtasAfterCbo(ASTNode newAst) throws SemanticException { + // analyzeCreateTable uses this.ast, but doPhase1 doesn't, so only reset it here. + this.ast = newAst; + newAst = analyzeCreateTable(newAst, qb, null); + if (newAst == null) { + LOG.error("analyzeCreateTable failed to initialize CTAS after CBO;" + + " new ast is " + this.ast.dump()); + throw new SemanticException("analyzeCreateTable failed to initialize CTAS after CBO"); + } + return newAst; + } + + private boolean canHandleAstForCbo(ASTNode ast, QB qb, PreCboCtx cboCtx) { + int root = ast.getToken().getType(); + boolean needToLogMessage = LOG.isInfoEnabled(); + boolean isSupportedRoot = + root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN || qb.isCTAS(); + // Check AST. + // Assumption: If top level QB is query then everything below it must also be Query + // Can there be an insert or CTAS that wouldn't + // be supported and would require additional checks similar to IsQuery? + boolean isSupportedType = + qb.getIsQuery() || qb.isCTAS() || cboCtx.type == PreCboCtx.Type.INSERT; + boolean result = isSupportedRoot && isSupportedType && createVwDesc == null; + if (!result) { + String msg = ""; + if (needToLogMessage) { + if (!isSupportedRoot) msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; "; + if (!isSupportedType) msg += "is not a query, CTAS, or insert; "; + if (createVwDesc != null) msg += "has create view; "; + + if (msg.isEmpty()) msg += "has some unspecified limitations; "; + LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2)); + } + return false; + } + // Now check QB in more detail. canHandleQbForCbo returns null if query can be handled. + String msg = canHandleQbForCbo(qb, true, needToLogMessage); + if (msg == null) { + return true; + } + if (needToLogMessage) { + LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2)); + } + return false; + } + + private class ASTSearcher { + private final LinkedList searchQueue = new LinkedList(); + /** + * Performs breadth-first search of the AST for a nested set of tokens. Tokens don't have to be + * each others' direct children, they can be separated by layers of other tokens. For each token + * in the list, the first one found is matched and there's no backtracking; thus, if AST has + * multiple instances of some token, of which only one matches, it is not guaranteed to be found. + * We use this for simple things. + * Not thread-safe - reuses searchQueue. + */ + public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) { + searchQueue.clear(); + searchQueue.add(ast); + for (int i = 0; i < tokens.length; ++i) { + boolean found = false; + int token = tokens[i]; + while (!searchQueue.isEmpty() && !found) { + ASTNode next = searchQueue.poll(); + found = next.getType() == token; + if (found) { + if (i == tokens.length - 1) return next; + searchQueue.clear(); + } + for (int j = 0; j < next.getChildCount(); ++j) { + searchQueue.add((ASTNode)next.getChild(j)); + } + } + if (!found) return null; + } + return null; + } + } + + private void replaceASTChild(ASTNode child, ASTNode newChild) { + ASTNode parent = (ASTNode)child.parent; + int childIndex = child.childIndex; + parent.deleteChild(childIndex); + parent.insertChild(childIndex, newChild); + } + private void putAccessedColumnsToReadEntity(HashSet inputs, ColumnAccessInfo columnAccessInfo) { Map> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap(); if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) { @@ -10576,8 +10745,8 @@ public RowResolver getRowResolver(Operator opt) { * the semantic analyzer need to deal with the select statement with respect * to the SerDe and Storage Format. */ - private ASTNode analyzeCreateTable(ASTNode ast, QB qb) - throws SemanticException { + private ASTNode analyzeCreateTable( + ASTNode ast, QB qb, PreCboCtx cboCtx) throws SemanticException { String[] qualifiedTabName = getQualifiedTableName((ASTNode) ast.getChild(0)); String dbDotTab = getDotName(qualifiedTabName); @@ -10667,6 +10836,9 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg()); } command_type = CTAS; + if (cboCtx != null) { + cboCtx.set(PreCboCtx.Type.CTAS, child); + } selectStmt = child; break; case HiveParser.TOK_TABCOLLIST: @@ -12238,27 +12410,44 @@ protected boolean deleting() { /**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/ - /* - * Entry point to Optimizations using Optiq. + /** + * Entry point to Optimizations using Optiq. Checks whether Optiq can handle the query. + * @param qbToChk Query block to check. + * @param verbose Whether return value should be verbose in case of failure. + * @return null if the query can be handled; non-null reason string if it cannot be. */ - private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { - boolean runOptiqPlanner = false; + private String canHandleQbForCbo(QB qbToChk, boolean topLevelQB, boolean verbose) { // Assumption: // 1. If top level QB is query then everything below it must also be Query // 2. Nested Subquery will return false for qbToChk.getIsQuery() - if ((!topLevelQB || qbToChk.getIsQuery()) - && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict")) - && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST)) - && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy() - && !queryProperties.hasSortBy() && !queryProperties.hasPTF() - && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery() - && !queryProperties.hasLateralViews()) { - runOptiqPlanner = true; - } else { - LOG.info("Can not invoke CBO; query contains operators not supported for CBO."); - } - - return runOptiqPlanner; + boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST); + boolean isStrictTest = isInTest + && !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict"); + boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest; + if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy() + && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy() + && !queryProperties.hasPTF() && !queryProperties.usesScript() + && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) { + return null; // Ok to run CBO. + } + + // Not ok to run CBO, build error message. + String msg = ""; + if (verbose) { + if (isStrictTest) msg += "is in test running in mode other than nonstrict; "; + if (!hasEnoughJoins) msg += "has too few joins; "; + if (queryProperties.hasClusterBy()) msg += "has cluster by; "; + if (queryProperties.hasDistributeBy()) msg += "has distribute by; "; + if (queryProperties.hasSortBy()) msg += "has sort by; "; + if (queryProperties.hasPTF()) msg += "has PTF; "; + if (queryProperties.usesScript()) msg += "uses scripts; "; + if (queryProperties.hasMultiDestQuery()) msg += "is a multi-destination query; "; + if (queryProperties.hasLateralViews()) msg += "has lateral views; "; + + if (msg.isEmpty()) msg += "has some unspecified limitations; "; + } + LOG.info("Can not invoke CBO; query contains operators not supported for CBO."); + return msg; } private class OptiqBasedPlanner implements Frameworks.PlannerAction { @@ -12974,7 +13163,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true); qbSQ.setSubQueryDef(subQuery.getSubQuery()); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1); + doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1, null); getMetaData(qbSQ); RelNode subQueryRelNode = genLogicalPlan(qbSQ, false); aliasToRel.put(subQuery.getAlias(), subQueryRelNode); @@ -13002,7 +13191,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery()); ctx_1 = initPhase1Ctx(); - doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1); + doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1, null); getMetaData(qbSQ_nic); RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false); aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode); @@ -14043,9 +14232,12 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticExcept // 0. Check if we can handle the query // This check is needed here because of SubQuery - if (!canHandleQuery(qb, false)) { - String msg = String.format("CBO Can not handle Sub Query"); - LOG.debug(msg); + String reason = canHandleQbForCbo(qb, false, LOG.isDebugEnabled()); + if (reason != null) { + String msg = "CBO can not handle Sub Query"; + if (LOG.isDebugEnabled()) { + LOG.debug(msg + " because it: " + reason); + } throw new OptiqSemanticException(msg); } diff --git ql/src/test/queries/clientpositive/cbo_correctness.q ql/src/test/queries/clientpositive/cbo_correctness.q index 4d8f156..30ec646 100644 --- ql/src/test/queries/clientpositive/cbo_correctness.q +++ ql/src/test/queries/clientpositive/cbo_correctness.q @@ -485,4 +485,4 @@ select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value fro -- Windowing select *, rank() over(partition by key order by value) as rr from src1; -select *, rank() over(partition by key order by value) from src1; \ No newline at end of file +select *, rank() over(partition by key order by value) from src1; diff --git ql/src/test/queries/clientpositive/ctas_colname.q ql/src/test/queries/clientpositive/ctas_colname.q index 5322626..890971e 100644 --- ql/src/test/queries/clientpositive/ctas_colname.q +++ ql/src/test/queries/clientpositive/ctas_colname.q @@ -3,9 +3,11 @@ -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src; -create table summary as select *, sum(key), count(value) from src; +create table summary as select *, key + 1, concat(value, value) from src limit 20; +create table summary as select *, key + 1, concat(value, value) from src limit 20; describe formatted summary; select * from summary; diff --git ql/src/test/queries/clientpositive/decimal_serde.q ql/src/test/queries/clientpositive/decimal_serde.q index cf3a86c..be7a4ac 100644 --- ql/src/test/queries/clientpositive/decimal_serde.q +++ ql/src/test/queries/clientpositive/decimal_serde.q @@ -15,12 +15,15 @@ SELECT * FROM DECIMAL_TEXT ORDER BY key, value; CREATE TABLE DECIMAL_RC STORED AS RCFile AS SELECT * FROM DECIMAL_TEXT; +describe formatted DECIMAL_RC; CREATE TABLE DECIMAL_LAZY_COL ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" STORED AS RCFile AS SELECT * FROM DECIMAL_RC; +describe formatted DECIMAL_LAZY_COL; + CREATE TABLE DECIMAL_SEQUENCE ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' diff --git ql/src/test/queries/clientpositive/insert0.q ql/src/test/queries/clientpositive/insert0.q new file mode 100644 index 0000000..36d01b6 --- /dev/null +++ ql/src/test/queries/clientpositive/insert0.q @@ -0,0 +1,38 @@ +set hive.cbo.enable=true; + +DROP TABLE insert_into1; +DROP TABLE ctas_table; +DROP TABLE ctas_part; + +CREATE TABLE insert_into1 (key int, value string); + +INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10; + +select * from insert_into1 order by key; + +INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10; + +select * from insert_into1 order by key; + +create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10; + +describe extended ctas_table; + +select * from ctas_table order by key; + + +set hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +create table ctas_part (key int, value string) partitioned by (modkey bigint); + +insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10; + +select * from ctas_part order by key; + + + +DROP TABLE insert_into1; +DROP TABLE ctas_table; +DROP TABLE ctas_part; \ No newline at end of file diff --git ql/src/test/results/clientpositive/ctas_colname.q.out ql/src/test/results/clientpositive/ctas_colname.q.out index 97dacf6..c26a66d 100644 --- ql/src/test/results/clientpositive/ctas_colname.q.out +++ ql/src/test/results/clientpositive/ctas_colname.q.out @@ -3,16 +3,20 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src +create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: -- SORT_QUERY_RESULTS -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src +create table summary as select *, key + 1, concat(value, value) from src limit 20 POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: Stage-1 is a root stage @@ -28,31 +32,27 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + expressions: key (type: string), value (type: string), (key + 1) (type: double), concat(value, value) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(key), count(value) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double), _col1 (type: bigint) + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: double), _col1 (type: bigint), _col0 (type: double), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -68,7 +68,7 @@ STAGE PLANS: Stage: Stage-3 Create Table Operator: Create Table - columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint + columns: key string, value string, _c1 double, _c2 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -77,12 +77,12 @@ STAGE PLANS: Stage: Stage-2 Stats-Aggr Operator -PREHOOK: query: create table summary as select *, sum(key), count(value) from src +PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@summary -POSTHOOK: query: create table summary as select *, sum(key), count(value) from src +POSTHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -95,10 +95,10 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@summary # col_name data_type comment -_col0 double -_col1 bigint +key string +value string _c1 double -_c2 bigint +_c2 string # Detailed Table Information Database: default @@ -110,9 +110,9 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows 1 - rawDataSize 25 - totalSize 26 + numRows 20 + rawDataSize 620 + totalSize 640 #### A masked pattern was here #### # Storage Information @@ -133,7 +133,26 @@ POSTHOOK: query: select * from summary POSTHOOK: type: QUERY POSTHOOK: Input: default@summary #### A masked pattern was here #### -130091.0 500 130091.0 500 +128 val_128 129.0 val_128val_128 +150 val_150 151.0 val_150val_150 +165 val_165 166.0 val_165val_165 +193 val_193 194.0 val_193val_193 +213 val_213 214.0 val_213val_213 +224 val_224 225.0 val_224val_224 +238 val_238 239.0 val_238val_238 +255 val_255 256.0 val_255val_255 +265 val_265 266.0 val_265val_265 +27 val_27 28.0 val_27val_27 +273 val_273 274.0 val_273val_273 +278 val_278 279.0 val_278val_278 +311 val_311 312.0 val_311val_311 +369 val_369 370.0 val_369val_369 +401 val_401 402.0 val_401val_401 +409 val_409 410.0 val_409val_409 +484 val_484 485.0 val_484val_484 +66 val_66 67.0 val_66val_66 +86 val_86 87.0 val_86val_86 +98 val_98 99.0 val_98val_98 PREHOOK: query: -- window functions explain create table x4 as select *, rank() over(partition by key order by value) as rr from src1 diff --git ql/src/test/results/clientpositive/decimal_serde.q.out ql/src/test/results/clientpositive/decimal_serde.q.out index e461c2e..d651799 100644 --- ql/src/test/results/clientpositive/decimal_serde.q.out +++ ql/src/test/results/clientpositive/decimal_serde.q.out @@ -96,6 +96,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@decimal_text POSTHOOK: Output: database:default POSTHOOK: Output: default@DECIMAL_RC +PREHOOK: query: describe formatted DECIMAL_RC +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@decimal_rc +POSTHOOK: query: describe formatted DECIMAL_RC +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@decimal_rc +# col_name data_type comment + +key decimal(10,0) +value int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 38 + rawDataSize 157 + totalSize 278 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" STORED AS RCFile AS @@ -112,6 +148,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@decimal_rc POSTHOOK: Output: database:default POSTHOOK: Output: default@DECIMAL_LAZY_COL +PREHOOK: query: describe formatted DECIMAL_LAZY_COL +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@decimal_lazy_col +POSTHOOK: query: describe formatted DECIMAL_LAZY_COL +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@decimal_lazy_col +# col_name data_type comment + +key decimal(10,0) +value int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 38 + rawDataSize 157 + totalSize 278 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' diff --git ql/src/test/results/clientpositive/insert0.q.out ql/src/test/results/clientpositive/insert0.q.out new file mode 100644 index 0000000..e83bae1 --- /dev/null +++ ql/src/test/results/clientpositive/insert0.q.out @@ -0,0 +1,208 @@ +PREHOOK: query: DROP TABLE insert_into1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE ctas_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE ctas_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE ctas_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE ctas_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into1 (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: CREATE TABLE insert_into1 (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into1 +PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from insert_into1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select * from insert_into1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from insert_into1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select * from insert_into1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +PREHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@ctas_table +POSTHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ctas_table +PREHOOK: query: describe extended ctas_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ctas_table +POSTHOOK: query: describe extended ctas_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ctas_table +key string +foo bigint + +#### A masked pattern was here #### +PREHOOK: query: select * from ctas_table order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_table +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_table order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_table +#### A masked pattern was here #### +0 3 +10 1 +100 2 +103 2 +104 2 +105 1 +11 1 +111 1 +113 2 +114 1 +PREHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ctas_part +POSTHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ctas_part +PREHOOK: query: insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ctas_part +POSTHOOK: query: insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ctas_part@modkey=1 +POSTHOOK: Output: default@ctas_part@modkey=2 +POSTHOOK: Output: default@ctas_part@modkey=3 +POSTHOOK: Output: default@ctas_part@modkey=4 +POSTHOOK: Output: default@ctas_part@modkey=5 +POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from ctas_part order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_part +PREHOOK: Input: default@ctas_part@modkey=1 +PREHOOK: Input: default@ctas_part@modkey=2 +PREHOOK: Input: default@ctas_part@modkey=3 +PREHOOK: Input: default@ctas_part@modkey=4 +PREHOOK: Input: default@ctas_part@modkey=5 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_part order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_part +POSTHOOK: Input: default@ctas_part@modkey=1 +POSTHOOK: Input: default@ctas_part@modkey=2 +POSTHOOK: Input: default@ctas_part@modkey=3 +POSTHOOK: Input: default@ctas_part@modkey=4 +POSTHOOK: Input: default@ctas_part@modkey=5 +#### A masked pattern was here #### +27 val_27 1 +86 val_86 1 +98 val_98 1 +165 val_165 2 +238 val_238 3 +255 val_255 3 +278 val_278 3 +311 val_311 4 +409 val_409 5 +484 val_484 5 +PREHOOK: query: DROP TABLE insert_into1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into1 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: DROP TABLE insert_into1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into1 +POSTHOOK: Output: default@insert_into1 +PREHOOK: query: DROP TABLE ctas_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_table +PREHOOK: Output: default@ctas_table +POSTHOOK: query: DROP TABLE ctas_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_table +POSTHOOK: Output: default@ctas_table +PREHOOK: query: DROP TABLE ctas_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_part +PREHOOK: Output: default@ctas_part +POSTHOOK: query: DROP TABLE ctas_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_part +POSTHOOK: Output: default@ctas_part