diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteViewSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteViewSemanticException.java new file mode 100644 index 0000000..3d27bec --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteViewSemanticException.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Exception from SemanticAnalyzer. + */ + +public class CalciteViewSemanticException extends SemanticException { + + private static final long serialVersionUID = 1L; + + public CalciteViewSemanticException() { + super(); + } + + public CalciteViewSemanticException(String message) { + super(message); + } + + public CalciteViewSemanticException(Throwable cause) { + super(cause); + } + + public CalciteViewSemanticException(String message, Throwable cause) { + super(message, cause); + } + + public CalciteViewSemanticException(ErrorMsg errorMsg, String... msgArgs) { + super(errorMsg, msgArgs); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 96ff5df..e7687be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -139,6 +139,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature; +import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteViewSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; @@ -300,7 +301,7 @@ public RelNode genLogicalPlan(ASTNode ast) throws SemanticException { return null; } ASTNode queryForCbo = ast; - if (cboCtx.type == PreCboCtx.Type.CTAS_OR_MV) { + if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) { queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx); @@ -331,7 +332,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // table, destination), so if the query is otherwise ok, it is as if we // did remove those and gave CBO the proper AST. That is kinda hacky. ASTNode queryForCbo = ast; - if (cboCtx.type == PreCboCtx.Type.CTAS_OR_MV) { + if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) { queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx); @@ -348,6 +349,9 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept try { if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) { + throw new SemanticException("Create view is not supported in cbo return path."); + } sinkOp = getOptimizedHiveOPDag(); LOG.info("CBO Succeeded; optimized logical plan."); this.ctx.setCboInfo("Plan optimized by CBO."); @@ -360,13 +364,20 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept newAST = fixUpAfterCbo(ast, newAST, cboCtx); // 2. Regen OP plan from optimized AST - init(false); - if (cboCtx.type == PreCboCtx.Type.CTAS_OR_MV) { - // Redo create-table/view analysis, because it's not part of doPhase1. - if (materializedView) { + if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) { + try { + handleCreateViewDDL(newAST); + } catch (SemanticException e) { + throw new CalciteViewSemanticException(e.getMessage()); + } + } else { + init(false); + if (cboCtx.type == PreCboCtx.Type.VIEW && materializedView) { + // Redo create-table/view analysis, because it's not part of + // doPhase1. // Use the REWRITTEN AST setAST(newAST); - newAST = reAnalyzeMaterializedViewAfterCbo(newAST); + newAST = reAnalyzeViewAfterCbo(newAST); // Store text of the ORIGINAL QUERY String originalText = ctx.getTokenRewriteStream().toString( cboCtx.nodeOfInterest.getTokenStartIndex(), @@ -375,7 +386,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept viewSelect = newAST; viewsExpanded = new ArrayList<>(); viewsExpanded.add(createVwDesc.getViewName()); - } else { + } else if (cboCtx.type == PreCboCtx.Type.CTAS) { // CTAS setAST(newAST); newAST = reAnalyzeCTASAfterCbo(newAST); @@ -427,6 +438,11 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // so avoid executing subqueries on non-cbo throw new SemanticException(e); } + else if( e instanceof CalciteViewSemanticException) { + // non-cbo path retries to execute create view and + // we believe it will throw the same error message + throw new SemanticException(e); + } else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats || e instanceof CalciteSemanticException ) { reAnalyzeAST = true; @@ -465,6 +481,21 @@ else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats return sinkOp; } + private void handleCreateViewDDL(ASTNode newAST) throws SemanticException { + saveViewDefinition(); + String originalText = createVwDesc.getViewOriginalText(); + String expandedText = createVwDesc.getViewExpandedText(); + List schema = createVwDesc.getSchema(); + List partitionColumns = createVwDesc.getPartCols(); + init(false); + setAST(newAST); + newAST = reAnalyzeViewAfterCbo(newAST); + createVwDesc.setViewOriginalText(originalText); + createVwDesc.setViewExpandedText(expandedText); + createVwDesc.setSchema(schema); + createVwDesc.setPartCols(partitionColumns); + } + /* * Tries to optimize FROM clause of multi-insert. No attempt to optimize insert clauses of the query. * Returns true if rewriting is successful, false otherwise. @@ -651,9 +682,7 @@ boolean canCBOHandleAst(ASTNode ast, QB qb, PreCboCtx cboCtx) { || qb.isCTAS() || qb.isMaterializedView() || cboCtx.type == PreCboCtx.Type.INSERT || cboCtx.type == PreCboCtx.Type.MULTI_INSERT; boolean noBadTokens = HiveCalciteUtil.validateASTForUnsupportedTokens(ast); - boolean result = isSupportedRoot && isSupportedType - && (getCreateViewDesc() == null || getCreateViewDesc().isMaterialized()) - && noBadTokens; + boolean result = isSupportedRoot && isSupportedType && noBadTokens; if (!result) { if (needToLogMessage) { @@ -665,9 +694,6 @@ boolean canCBOHandleAst(ASTNode ast, QB qb, PreCboCtx cboCtx) { msg += "is not a query with at least one source table " + " or there is a subquery without a source table, or CTAS, or insert; "; } - if (getCreateViewDesc() != null && !getCreateViewDesc().isMaterialized()) { - msg += "has create view; "; - } if (!noBadTokens) { msg += "has unsupported tokens; "; } @@ -835,7 +861,7 @@ String fixCtasColumnName(String colName) { */ static class PreCboCtx extends PlannerContext { enum Type { - NONE, INSERT, MULTI_INSERT, CTAS_OR_MV, UNEXPECTED + NONE, INSERT, MULTI_INSERT, CTAS, VIEW, UNEXPECTED } private ASTNode nodeOfInterest; @@ -853,8 +879,13 @@ private void set(Type type, ASTNode ast) { } @Override - void setCTASOrMVToken(ASTNode child) { - set(PreCboCtx.Type.CTAS_OR_MV, child); + void setCTASToken(ASTNode child) { + set(PreCboCtx.Type.CTAS, child); + } + + @Override + void setViewToken(ASTNode child) { + set(PreCboCtx.Type.VIEW, child); } @Override @@ -884,7 +915,8 @@ ASTNode fixUpAfterCbo(ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) // nothing to do return newAst; - case CTAS_OR_MV: { + case CTAS: + case VIEW: { // Patch the optimized query back into original CTAS AST, replacing the // original query. replaceASTChild(cboCtx.nodeOfInterest, newAst); @@ -927,7 +959,7 @@ ASTNode reAnalyzeCTASAfterCbo(ASTNode newAst) throws SemanticException { return newAst; } - ASTNode reAnalyzeMaterializedViewAfterCbo(ASTNode newAst) throws SemanticException { + ASTNode reAnalyzeViewAfterCbo(ASTNode newAst) throws SemanticException { // analyzeCreateView uses this.ast, but doPhase1 doesn't, so only reset it // here. newAst = analyzeCreateView(newAst, getQB(), null); @@ -1245,9 +1277,11 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); try { calciteGenPlan = genLogicalPlan(getQB(), true, null, null); + // if it is to create view, we do not use table alias resultSchema = SemanticAnalyzer.convertRowSchemaToResultSetSchema( relToHiveRR.get(calciteGenPlan), - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); + getQB().isView() ? false : HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); } catch (SemanticException e) { semanticException = e; throw new RuntimeException(e); @@ -1848,6 +1882,10 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ RexNode calciteJoinCond = null; if (joinCond != null) { JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); + RowResolver input = RowResolver.getCombinedRR(leftRR, rightRR); + if (unparseTranslator != null && unparseTranslator.isEnabled()) { + genAllExprNodeDesc(joinCond, input, jCtx); + } Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, jCtx); if (jCtx.getError() != null) { @@ -2809,8 +2847,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 4. Construct GB Keys (ExprNode) for (int i = 0; i < grpByAstExprs.size(); ++i) { ASTNode grpbyExpr = grpByAstExprs.get(i); - Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); + Map astToExprNDescMap = genAllExprNodeDesc(grpbyExpr, groupByInputRowResolver); ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); if (grpbyExprNDesc == null) throw new CalciteSemanticException("Invalid Column Reference: " + grpbyExpr.dump(), @@ -2959,8 +2996,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException obASTExpr = (ASTNode) obASTExprLst.get(i); nullObASTExpr = (ASTNode) obASTExpr.getChild(0); ASTNode ref = (ASTNode) nullObASTExpr.getChild(0); - Map astToExprNDescMap = TypeCheckProcFactory.genExprNode( - obASTExpr, new TypeCheckCtx(inputRR)); + Map astToExprNDescMap = genAllExprNodeDesc(ref, inputRR); ExprNodeDesc obExprNDesc = astToExprNDescMap.get(ref); if (obExprNDesc == null) throw new SemanticException("Invalid order by expression: " + obASTExpr.toString()); @@ -3442,6 +3478,9 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) LOG.debug("Find UDTF " + funcName); genericUDTF = fi.getGenericUDTF(); genericUDTFName = funcName; + if (!fi.isNative()) { + unparseTranslator.addIdentifierTranslation((ASTNode) expr.getChild(0)); + } if (genericUDTF != null && (selectStar = exprType == HiveParser.TOK_FUNCTIONSTAR)) { genColListRegex(".*", null, (ASTNode) expr.getChild(0), col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); @@ -3467,12 +3506,15 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) switch (selExprChild.getType()) { case HiveParser.Identifier: udtfColAliases.add(unescapeIdentifier(selExprChild.getText().toLowerCase())); + unparseTranslator.addIdentifierTranslation(selExprChild); break; case HiveParser.TOK_TABALIAS: assert (selExprChild.getChildCount() == 1); udtfTableAlias = unescapeIdentifier(selExprChild.getChild(0) .getText()); qb.addAlias(udtfTableAlias); + unparseTranslator.addIdentifierTranslation((ASTNode) selExprChild + .getChild(0)); break; default: throw new SemanticException("Find invalid token type " + selExprChild.getType() @@ -3521,6 +3563,10 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) inputRR, autogenColAliasPrfxIncludeFuncName(), i); tabAlias = colRef[0]; colAlias = colRef[1]; + if (hasAsClause) { + unparseTranslator.addIdentifierTranslation((ASTNode) child + .getChild(1)); + } } // 6.4 Build ExprNode corresponding to colums diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 59d537f..bfe2ab5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -420,6 +420,10 @@ public boolean isMaterializedView() { return viewDesc != null && viewDesc.isMaterialized(); } + public boolean isView() { + return viewDesc != null && !viewDesc.isMaterialized(); + } + void addEncryptedTargetTablePath(Path p) { if(encryptedTargetTablePaths == null) { encryptedTargetTablePaths = new ArrayList<>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dceb4a5..03ab0c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -10720,7 +10720,10 @@ void setParseTreeAttr(ASTNode child, Phase1Ctx ctx_1) { this.ctx_1 = ctx_1; } - void setCTASOrMVToken(ASTNode child) { + void setCTASToken(ASTNode child) { + } + + void setViewToken(ASTNode child) { } void setInsertToken(ASTNode ast, boolean isTmpFileDest) { @@ -11033,7 +11036,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce } // 3. Deduce Resultset Schema - if (createVwDesc != null) { + if (createVwDesc != null && !this.ctx.isCboSucceeded()) { resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver()); } else { // resultSchema will be null if @@ -11064,7 +11067,10 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce if (ctx.getExplainAnalyze() == AnalyzeState.RUNNING) { return; } - saveViewDefinition(); + + if (!ctx.isCboSucceeded()) { + saveViewDefinition(); + } // validate the create view statement at this point, the createVwDesc gets // all the information for semanticcheck @@ -11232,7 +11238,7 @@ private void enforceScanLimits(ParseContext pCtx, FetchTask fTask) return resultSchema; } - private void saveViewDefinition() throws SemanticException { + protected void saveViewDefinition() throws SemanticException { // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List derivedSchema = @@ -11524,6 +11530,11 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) continue; } String[] tmp = input.reverseLookup(columnDesc.getColumn()); + // in subquery case, tmp may be from outside. + if (tmp[0] != null && columnDesc.getTabAlias() != null + && !tmp[0].equals(columnDesc.getTabAlias()) && tcCtx.getOuterRR() != null) { + tmp = tcCtx.getOuterRR().reverseLookup(columnDesc.getColumn()); + } StringBuilder replacementText = new StringBuilder(); replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf)); replacementText.append("."); @@ -11807,7 +11818,7 @@ ASTNode analyzeCreateTable( } command_type = CTAS; if (plannerCtx != null) { - plannerCtx.setCTASOrMVToken(child); + plannerCtx.setCTASToken(child); } selectStmt = child; break; @@ -12084,7 +12095,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt case HiveParser.TOK_QUERY: // For CBO if (plannerCtx != null) { - plannerCtx.setCTASOrMVToken(child); + plannerCtx.setViewToken(child); } selectStmt = child; break; @@ -12144,7 +12155,6 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt storageFormat.getSerdeProps()); addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW); queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); - qb.setViewDesc(createVwDesc); } else { createVwDesc = new CreateViewDesc( dbDotTable, cols, comment, tblProps, partColNames, @@ -12155,6 +12165,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); queryState.setCommandType(HiveOperation.CREATEVIEW); } + qb.setViewDesc(createVwDesc); return selectStmt; } diff --git a/ql/src/test/queries/clientpositive/view_cbo.q b/ql/src/test/queries/clientpositive/view_cbo.q new file mode 100644 index 0000000..4172678 --- /dev/null +++ b/ql/src/test/queries/clientpositive/view_cbo.q @@ -0,0 +1,72 @@ +set hive.mapred.mode=nonstrict; +-- SORT_QUERY_RESULTS + +explain +select key, value, avg(key + 1) from src +group by value, key with rollup +order by key, value limit 20; + +drop view v; +create view v as +with q1 as ( select key from src where key = '5') +select * from q1; +desc formatted v; + +drop view v; +create view v as +select b.key, count(*) as c +from src b +group by b.key +having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +; +desc formatted v; + +drop view v; +create view v as +select * +from src b +where not exists + (select distinct a.key + from src a + where b.value = a.value and a.value > 'val_2' + ) +; +desc formatted v; + +drop view v; +create view v as select a.key from src a join src b on a.key=b.key; +desc formatted v; + +CREATE VIEW view15 AS +SELECT key,COUNT(value) AS value_count +FROM src +GROUP BY key; +desc formatted view15; + +CREATE VIEW view16 AS +SELECT DISTINCT value +FROM src; + +desc formatted view16; + +drop view v; +create view v as select key from src; +desc formatted v; + +drop view v; +create view v as select * from src; +desc formatted v; + +drop view v; +create view v as select * from src intersect select * from src; +desc formatted v; + +drop view v; +create view v as select * from src except select * from src; +desc formatted v; + +explain select * from v; diff --git a/ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out b/ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out index 2d97755..9496e52 100644 --- a/ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out +++ b/ql/src/test/results/clientnegative/selectDistinctStarNeg_1.q.out @@ -2,4 +2,4 @@ PREHOOK: query: drop view if exists v PREHOOK: type: DROPVIEW POSTHOOK: query: drop view if exists v POSTHOOK: type: DROPVIEW -FAILED: SemanticException [Error 10036]: Duplicate column name: key +FAILED: SemanticException org.apache.hadoop.hive.ql.optimizer.calcite.CalciteViewSemanticException: Duplicate column name: key diff --git a/ql/src/test/results/clientpositive/create_view.q.out b/ql/src/test/results/clientpositive/create_view.q.out index 7deac3e..d3b858a 100644 --- a/ql/src/test/results/clientpositive/create_view.q.out +++ b/ql/src/test/results/clientpositive/create_view.q.out @@ -162,10 +162,10 @@ POSTHOOK: query: EXPLAIN CREATE VIEW view0(valoo) AS SELECT upper(value) FROM src WHERE key=86 POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false @@ -720,7 +720,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@table1 POSTHOOK: Output: database:default POSTHOOK: Output: default@view8 -POSTHOOK: Lineage: view8.c EXPRESSION [] +POSTHOOK: Lineage: view8.c SIMPLE [] PREHOOK: query: DESCRIBE EXTENDED view8 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view8 @@ -1279,8 +1279,8 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@view14 POSTHOOK: Lineage: view14.k1 EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: view14.k2 EXPRESSION [(src)s4.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: view14.v1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: view14.v2 EXPRESSION [(src)s3.null, (src)s4.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: view14.v1 EXPRESSION [(src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: view14.v2 EXPRESSION [(src)s4.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE EXTENDED view14 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view14 diff --git a/ql/src/test/results/clientpositive/create_view_translate.q.out b/ql/src/test/results/clientpositive/create_view_translate.q.out index 78614f7..db9fd78 100644 --- a/ql/src/test/results/clientpositive/create_view_translate.q.out +++ b/ql/src/test/results/clientpositive/create_view_translate.q.out @@ -131,10 +131,10 @@ POSTHOOK: query: explain CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false diff --git a/ql/src/test/results/clientpositive/cte_2.q.out b/ql/src/test/results/clientpositive/cte_2.q.out index c2bc565..4c9af87 100644 --- a/ql/src/test/results/clientpositive/cte_2.q.out +++ b/ql/src/test/results/clientpositive/cte_2.q.out @@ -118,7 +118,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: select * from v1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -154,7 +154,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: with q1 as ( select key from src where key = '4') select * from v1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/cte_4.q.out b/ql/src/test/results/clientpositive/cte_4.q.out index ce92dca..dbe4140 100644 --- a/ql/src/test/results/clientpositive/cte_4.q.out +++ b/ql/src/test/results/clientpositive/cte_4.q.out @@ -148,7 +148,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: select * from v1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -184,7 +184,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: with q1 as ( select key from src where key = '4') select * from v1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/explain_ddl.q.out b/ql/src/test/results/clientpositive/explain_ddl.q.out index ee2a60a..e108e22 100644 --- a/ql/src/test/results/clientpositive/explain_ddl.q.out +++ b/ql/src/test/results/clientpositive/explain_ddl.q.out @@ -417,10 +417,10 @@ PREHOOK: type: CREATEVIEW POSTHOOK: query: EXPLAIN CREATE VIEW V1 AS select * from M1 POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false diff --git a/ql/src/test/results/clientpositive/explain_dependency.q.out b/ql/src/test/results/clientpositive/explain_dependency.q.out index 6ab41c7..070da56 100644 --- a/ql/src/test/results/clientpositive/explain_dependency.q.out +++ b/ql/src/test/results/clientpositive/explain_dependency.q.out @@ -128,7 +128,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V5 -POSTHOOK: Lineage: V5.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: V5.ds SIMPLE [] POSTHOOK: Lineage: V5.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] POSTHOOK: Lineage: V5.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: V5.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] diff --git a/ql/src/test/results/clientpositive/explain_logical.q.out b/ql/src/test/results/clientpositive/explain_logical.q.out index d1176c1..fb91a3b 100644 --- a/ql/src/test/results/clientpositive/explain_logical.q.out +++ b/ql/src/test/results/clientpositive/explain_logical.q.out @@ -472,7 +472,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V5 -POSTHOOK: Lineage: V5.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: V5.ds SIMPLE [] POSTHOOK: Lineage: V5.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] POSTHOOK: Lineage: V5.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: V5.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] diff --git a/ql/src/test/results/clientpositive/llap/cbo_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_views.q.out index 2ac2d3a..44e5501 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_views.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_views.q.out @@ -173,7 +173,7 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Output: database:default POSTHOOK: Output: default@v4 POSTHOOK: Lineage: v4.c_int SIMPLE [(cbo_t1)cbo_t1.FieldSchema(name:c_int, type:int, comment:null), ] -POSTHOOK: Lineage: v4.key SIMPLE [(cbo_t1)cbo_t1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: v4.key SIMPLE [] PREHOOK: query: with q1 as ( select c_int from q2 where c_boolean = false), q2 as ( select c_int,c_boolean from v1 where value = '1') select sum(c_int) from (select c_int from q1) a diff --git a/ql/src/test/results/clientpositive/llap/cte_2.q.out b/ql/src/test/results/clientpositive/llap/cte_2.q.out index ef66c40..16906c1 100644 --- a/ql/src/test/results/clientpositive/llap/cte_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_2.q.out @@ -118,7 +118,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: select * from v1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -154,7 +154,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: with q1 as ( select key from src where key = '4') select * from v1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/cte_4.q.out b/ql/src/test/results/clientpositive/llap/cte_4.q.out index ce92dca..dbe4140 100644 --- a/ql/src/test/results/clientpositive/llap/cte_4.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_4.q.out @@ -148,7 +148,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: select * from v1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -184,7 +184,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v1 -POSTHOOK: Lineage: v1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1.key SIMPLE [] PREHOOK: query: with q1 as ( select key from src where key = '4') select * from v1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 621f337..21fd10c 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -5178,9 +5178,9 @@ sum(p_retailprice) as s from part group by p_mfgr, p_brand POSTHOOK: type: CREATEVIEW -Plan not optimized by CBO. +Plan optimized by CBO. -Stage-0 +Stage-1 Create View Operator: name:default.mfgr_price_view,original text:select p_mfgr, p_brand, sum(p_retailprice) as s diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 495ad09..2f53e60 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -297,7 +297,7 @@ PREHOOK: type: CREATEVIEW PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@dest_v3 -{"version":"1.0","engine":"tez","database":"default","hash":"a0c2481ce1c24895a43a950f93a10da7","queryText":"create view dest_v3 (a1, a2, a3, a4, a5, a6, a7) as\n select x.csmallint, x.cbigint bint1, x.ctinyint, c.cbigint bint2, x.cint, x.cfloat, c.cstring1\n from alltypesorc c\n join (\n select a.csmallint csmallint, a.ctinyint ctinyint, a.cstring2 cstring2,\n a.cint cint, a.cstring1 ctring1, b.cfloat cfloat, b.cbigint cbigint\n from ( select * from alltypesorc a where cboolean1=true ) a\n join alltypesorc b on (a.csmallint = b.cint)\n ) x on (x.ctinyint = c.cbigint)\n where x.csmallint=11\n and x.cint > 899\n and x.cfloat > 4.5\n and c.cstring1 < '7'\n and x.cint + x.cfloat + length(c.cstring1) < 1000","edges":[{"sources":[7],"targets":[0],"edgeType":"PROJECTION"},{"sources":[8],"targets":[1,2],"edgeType":"PROJECTION"},{"sources":[9],"targets":[3],"edgeType":"PROJECTION"},{"sources":[10],"targets":[4],"edgeType":"PROJECTION"},{"sources":[11],"targets":[5],"edgeType":"PROJECTION"},{"sources":[12],"targets":[6],"edgeType":"PROJECTION"},{"sources":[8,9],"targets":[0,1,3,2,4,5,6],"expression":"(c.cbigint = UDFToLong(x._col1))","edgeType":"PREDICATE"},{"sources":[13],"targets":[0,1,3,2,4,5,6],"expression":"(a.cboolean1 = true)","edgeType":"PREDICATE"},{"sources":[7,10],"targets":[0,1,3,2,4,5,6],"expression":"(UDFToInteger(a._col1) = b.cint)","edgeType":"PREDICATE"},{"sources":[7,10,11,12],"targets":[0,1,3,2,4,5,6],"expression":"((x.csmallint = 11) and (x.cint > 899) and (x.cfloat > 4.5) and (c.cstring1 < '7') and (((x.cint + x.cfloat) + length(c.cstring1)) < 1000))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_v3.csmallint"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_v3.bint1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_v3.bint2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_v3.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_v3.cint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_v3.cfloat"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_v3.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":10,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":11,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"},{"id":12,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":13,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"a0c2481ce1c24895a43a950f93a10da7","queryText":"create view dest_v3 (a1, a2, a3, a4, a5, a6, a7) as\n select x.csmallint, x.cbigint bint1, x.ctinyint, c.cbigint bint2, x.cint, x.cfloat, c.cstring1\n from alltypesorc c\n join (\n select a.csmallint csmallint, a.ctinyint ctinyint, a.cstring2 cstring2,\n a.cint cint, a.cstring1 ctring1, b.cfloat cfloat, b.cbigint cbigint\n from ( select * from alltypesorc a where cboolean1=true ) a\n join alltypesorc b on (a.csmallint = b.cint)\n ) x on (x.ctinyint = c.cbigint)\n where x.csmallint=11\n and x.cint > 899\n and x.cfloat > 4.5\n and c.cstring1 < '7'\n and x.cint + x.cfloat + length(c.cstring1) < 1000","edges":[{"sources":[],"targets":[0],"expression":"11","edgeType":"PROJECTION"},{"sources":[7],"targets":[1,2],"edgeType":"PROJECTION"},{"sources":[8],"targets":[3],"edgeType":"PROJECTION"},{"sources":[9],"targets":[4],"edgeType":"PROJECTION"},{"sources":[10],"targets":[5],"edgeType":"PROJECTION"},{"sources":[11],"targets":[6],"edgeType":"PROJECTION"},{"sources":[11,7],"targets":[0,1,3,2,4,5,6],"expression":"((c.cstring1 < '7') and c.cbigint is not null)","edgeType":"PREDICATE"},{"sources":[7,8],"targets":[0,1,3,2,4,5,6],"expression":"(c.cbigint = UDFToLong(a.ctinyint))","edgeType":"PREDICATE"},{"sources":[10,9],"targets":[0,1,3,2,4,5,6],"expression":"((b.cfloat > 4.5) and (11 = b.cint))","edgeType":"PREDICATE"},{"sources":[12,13,9,8],"targets":[0,1,3,2,4,5,6],"expression":"(a.cboolean1 and (a.csmallint = 11) and (a.cint > 899) and a.ctinyint is not null)","edgeType":"PREDICATE"},{"sources":[9,10,11],"targets":[0,1,3,2,4,5,6],"expression":"(((UDFToFloat(a.cint) + b.cfloat) + UDFToFloat(length(c.cstring1))) < 1000.0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_v3.csmallint"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_v3.bint1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_v3.bint2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_v3.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_v3.cint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_v3.cfloat"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_v3.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":10,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"},{"id":11,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":12,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":13,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"}]} PREHOOK: query: alter view dest_v3 as select * from ( select sum(a.ctinyint) over (partition by a.csmallint order by a.csmallint) a, diff --git a/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out b/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out index afb9c4f..c5ca782 100644 --- a/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out +++ b/ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out @@ -1343,10 +1343,10 @@ PREHOOK: type: CREATEVIEW POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false @@ -3807,10 +3807,10 @@ PREHOOK: type: CREATEVIEW POSTHOOK: query: explain create view sdi as select distinct * from src order by key limit 2 POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index 185f08e..d3fdec0 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -807,7 +807,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@cv3 -POSTHOOK: Lineage: cv3._c2 EXPRESSION [(src)b.null, ] +POSTHOOK: Lineage: cv3._c2 EXPRESSION [(src)b.null, (src)src.null, ] POSTHOOK: Lineage: cv3.key SIMPLE [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: cv3.value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: describe extended cv3 diff --git a/ql/src/test/results/clientpositive/llap/union_top_level.q.out b/ql/src/test/results/clientpositive/llap/union_top_level.q.out index de23f48..52926b6 100644 --- a/ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ b/ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -1100,10 +1100,10 @@ union all select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false diff --git a/ql/src/test/results/clientpositive/perf/query14.q.out b/ql/src/test/results/clientpositive/perf/query14.q.out index 21cabbc..332d43a 100644 --- a/ql/src/test/results/clientpositive/perf/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product -Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product +Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 61' is a cross product Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 62' is a cross product PREHOOK: query: explain diff --git a/ql/src/test/results/clientpositive/spark/union_top_level.q.out b/ql/src/test/results/clientpositive/spark/union_top_level.q.out index 74719bb..6adf6c4 100644 --- a/ql/src/test/results/clientpositive/spark/union_top_level.q.out +++ b/ql/src/test/results/clientpositive/spark/union_top_level.q.out @@ -969,10 +969,10 @@ union all select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage STAGE PLANS: - Stage: Stage-0 + Stage: Stage-1 Create View Operator: Create View or replace: false diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 9f1a401..dd8849d 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -573,9 +573,9 @@ PREHOOK: type: CREATEVIEW POSTHOOK: query: explain analyze create view v as with cte as (select * from src order by key limit 5) select * from cte POSTHOOK: type: CREATEVIEW -Plan not optimized by CBO. +Plan optimized by CBO. -Stage-0 +Stage-1 Create View Operator: name:default.v,original text:with cte as (select * from src order by key limit 5) select * from cte diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 17c9ec3..ef71d73 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -435,9 +435,9 @@ PREHOOK: type: CREATEVIEW POSTHOOK: query: explain create view v as with cte as (select * from src order by key limit 5) select * from cte POSTHOOK: type: CREATEVIEW -Plan not optimized by CBO. +Plan optimized by CBO. -Stage-0 +Stage-1 Create View Operator: name:default.v,original text:with cte as (select * from src order by key limit 5) select * from cte diff --git a/ql/src/test/results/clientpositive/view_alias.q.out b/ql/src/test/results/clientpositive/view_alias.q.out index 0d7e72f..90bf28d 100644 --- a/ql/src/test/results/clientpositive/view_alias.q.out +++ b/ql/src/test/results/clientpositive/view_alias.q.out @@ -146,7 +146,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@v -POSTHOOK: Lineage: v._c2 SIMPLE [] +POSTHOOK: Lineage: v._c1 SIMPLE [] POSTHOOK: Lineage: v.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: v.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: desc formatted v @@ -159,7 +159,7 @@ POSTHOOK: Input: default@v key string value string -_c2 string +_c1 string # Detailed Table Information Database: default @@ -390,8 +390,8 @@ POSTHOOK: Input: default@b POSTHOOK: Output: database:default POSTHOOK: Output: default@v POSTHOOK: Lineage: v._c0 SIMPLE [] -POSTHOOK: Lineage: v._c3 SIMPLE [] -POSTHOOK: Lineage: v._c6 SIMPLE [] +POSTHOOK: Lineage: v._c2 SIMPLE [] +POSTHOOK: Lineage: v._c4 SIMPLE [] POSTHOOK: Lineage: v.ca SIMPLE [(a)a.FieldSchema(name:ca, type:string, comment:null), ] POSTHOOK: Lineage: v.caa SIMPLE [(a)a.FieldSchema(name:caa, type:string, comment:null), ] POSTHOOK: Lineage: v.cb SIMPLE [(b)b.FieldSchema(name:cb, type:string, comment:null), ] @@ -407,10 +407,10 @@ POSTHOOK: Input: default@v _c0 string ca string caa string -_c3 int +_c2 int cb string cbb string -_c6 int +_c4 int # Detailed Table Information Database: default diff --git a/ql/src/test/results/clientpositive/view_cbo.q.out b/ql/src/test/results/clientpositive/view_cbo.q.out new file mode 100644 index 0000000..97b7ca5 --- /dev/null +++ b/ql/src/test/results/clientpositive/view_cbo.q.out @@ -0,0 +1,823 @@ +PREHOOK: query: explain +select key, value, avg(key + 1) from src +group by value, key with rollup +order by key, value limit 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key, value, avg(key + 1) from src +group by value, key with rollup +order by key, value limit 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(_col2) + keys: _col0 (type: string), _col1 (type: string), 0 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: avg(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + pruneGroupingSetId: true + Select Operator + expressions: _col1 (type: string), _col0 (type: string), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: double) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key SIMPLE [] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: with q1 as ( select key from src where key = '5') + select * from q1 +View Expanded Text: with q1 as ( select `src`.`key` from `default`.`src` where `src`.`key` = '5') + select `q1`.`key` from q1 +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as +select b.key, count(*) as c +from src b +group by b.key +having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as +select b.key, count(*) as c +from src b +group by b.key +having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.c EXPRESSION [(src)b.null, ] +POSTHOOK: Lineage: v.key SIMPLE [(src)b.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +c bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select b.key, count(*) as c + from src b + group by b.key + having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +View Expanded Text: select `b`.`key`, count(*) as `c` + from `default`.`src` `b` + group by `b`.`key` + having exists + (select `a`.`key` + from `default`.`src` `a` + where `a`.`key` = b.key and `a`.`value` > 'val_9' + ) +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as +select * +from src b +where not exists + (select distinct a.key + from src a + where b.value = a.value and a.value > 'val_2' + ) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as +select * +from src b +where not exists + (select distinct a.key + from src a + where b.value = a.value and a.value > 'val_2' + ) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key SIMPLE [(src)b.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v.value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * + from src b + where not exists + (select distinct a.key + from src a + where b.value = a.value and a.value > 'val_2' + ) +View Expanded Text: select `b`.`key`, `b`.`value` + from `default`.`src` `b` + where not exists + (select distinct `a`.`key` + from `default`.`src` `a` + where `b`.`value` = `a`.`value` and `a`.`value` > 'val_2' + ) +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select a.key from src a join src b on a.key=b.key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select a.key from src a join src b on a.key=b.key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select a.key from src a join src b on a.key=b.key +View Expanded Text: select `a`.`key` from `default`.`src` `a` join `default`.`src` `b` on `a`.`key`=`b`.`key` +View Rewrite Enabled: No +PREHOOK: query: CREATE VIEW view15 AS +SELECT key,COUNT(value) AS value_count +FROM src +GROUP BY key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@view15 +POSTHOOK: query: CREATE VIEW view15 AS +SELECT key,COUNT(value) AS value_count +FROM src +GROUP BY key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@view15 +POSTHOOK: Lineage: view15.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: view15.value_count EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted view15 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@view15 +POSTHOOK: query: desc formatted view15 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@view15 +# col_name data_type comment + +key string +value_count bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT key,COUNT(value) AS value_count + FROM src + GROUP BY key +View Expanded Text: SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count` + FROM `default`.`src` + GROUP BY `src`.`key` +View Rewrite Enabled: No +PREHOOK: query: CREATE VIEW view16 AS +SELECT DISTINCT value +FROM src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@view16 +POSTHOOK: query: CREATE VIEW view16 AS +SELECT DISTINCT value +FROM src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@view16 +POSTHOOK: Lineage: view16.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted view16 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@view16 +POSTHOOK: query: desc formatted view16 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@view16 +# col_name data_type comment + +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT DISTINCT value + FROM src +View Expanded Text: SELECT DISTINCT `src`.`value` + FROM `default`.`src` +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select key from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key from src +View Expanded Text: select `src`.`key` from `default`.`src` +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select * from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select * from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from src +View Expanded Text: select `src`.`key`, `src`.`value` from `default`.`src` +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select * from src intersect select * from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select * from src intersect select * from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from src intersect select * from src +View Expanded Text: select `src`.`key`, `src`.`value` from `default`.`src` intersect select `src`.`key`, `src`.`value` from `default`.`src` +View Rewrite Enabled: No +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select * from src except select * from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select * from src except select * from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from src except select * from src +View Expanded Text: select `src`.`key`, `src`.`value` from `default`.`src` except select `src`.`key`, `src`.`value` from `default`.`src` +View Rewrite Enabled: No +PREHOOK: query: explain select * from v +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(2) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 2 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col2), sum(_col3) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col2), sum(_col3) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col2 > 0) and ((_col2 * 2) = _col3)) (type: boolean) + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +