diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index efe3286..f5210ec 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -295,7 +295,8 @@ protected static Hive createHiveDB(HiveConf conf) throws SemanticException { return idToTableNameMap; } - public abstract void analyzeInternal(ASTNode ast) throws SemanticException; + public abstract ASTNode analyzeInternal(ASTNode ast) throws SemanticException; + public void init() { //no-op } @@ -304,10 +305,10 @@ public void initCtx(Context ctx) { this.ctx = ctx; } - public void analyze(ASTNode ast, Context ctx) throws SemanticException { + public ASTNode analyze(ASTNode ast, Context ctx) throws SemanticException { initCtx(ctx); init(); - analyzeInternal(ast); + return analyzeInternal(ast); } public void validate() throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 5b77e6f..96619a0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -590,7 +590,7 @@ private void checkForPartitionColumns(List specifiedCols, List p } @Override - public void analyze(ASTNode ast, Context origCtx) throws SemanticException { + public ASTNode analyze(ASTNode ast, Context origCtx) throws SemanticException { QB qb; QBParseInfo qbp; @@ -618,5 +618,6 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { LOG.info("Invoking analyze on original query"); analyzeInternal(originalTree); } + return ast; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 713bd54..81ad5de 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -222,7 +222,7 @@ public DDLSemanticAnalyzer(HiveConf conf, Hive db) throws SemanticException { } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { switch (ast.getToken().getType()) { case HiveParser.TOK_ALTERTABLE_PARTITION: { @@ -451,6 +451,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { default: throw new SemanticException("Unsupported command."); } + return ast; } private void analyzeGrantRevokeRole(boolean grant, ASTNode ast) throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index eeee327..964305d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -42,7 +42,7 @@ public ExplainSemanticAnalyzer(HiveConf conf) throws SemanticException { @SuppressWarnings("unchecked") @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { boolean extended = false; boolean formatted = false; @@ -62,7 +62,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // Create a semantic analyzer for the query ASTNode input = (ASTNode) ast.getChild(0); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, input); - sem.analyze(input, ctx); + input = sem.analyze(input, ctx); sem.validate(); ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); @@ -95,6 +95,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { fieldList = explTask.getResultSchema(); rootTasks.add(explTask); + + return input; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java index be0ad62..1cd2808 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java @@ -48,7 +48,7 @@ public ExportSemanticAnalyzer(HiveConf conf) throws SemanticException { } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { Tree tableTree = ast.getChild(0); Tree toTree = ast.getChild(1); @@ -122,5 +122,6 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { inputs.add(new ReadEntity(ts.tableHandle)); } outputs.add(new WriteEntity(parentPath, toURI.getScheme().equals("hdfs"))); + return ast; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java index da917f7..5e5e348 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java @@ -41,7 +41,7 @@ public FunctionSemanticAnalyzer(HiveConf conf) throws SemanticException { } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { if (ast.getToken().getType() == HiveParser.TOK_CREATEFUNCTION) { analyzeCreateFunction(ast); } @@ -50,6 +50,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } LOG.info("analyze done"); + return ast; } private void analyzeCreateFunction(ASTNode ast) throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 5dff3fe..c537c34 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -1857,45 +1857,41 @@ unionType : KW_UNIONTYPE LESSTHAN colTypeList GREATERTHAN -> ^(TOK_UNIONTYPE colTypeList) ; -queryOperator -@init { msgs.push("query operator"); } +setOperator +@init { msgs.push("set operator"); } @after { msgs.pop(); } : KW_UNION KW_ALL -> ^(TOK_UNION) ; -// select statement select ... from ... where ... group by ... order by ... queryStatementExpression - : queryStatement (queryOperator^ queryStatement)* + : + fromStatement + | regularBody + ; + +fromStatement + : singleFromStatement (setOperator^ singleFromStatement)* ; -queryStatement +singleFromStatement : fromClause ( b+=body )+ -> ^(TOK_QUERY fromClause body+) - | regular_body ; -regular_body +regularBody : - insertClause - selectClause - fromClause - whereClause? - groupByClause? - havingClause? - orderByClause? - clusterByClause? - distributeByClause? - sortByClause? - window_clause? - limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause - selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? window_clause? limitClause?)) + insertClause^ + selectStatement | selectStatement ; selectStatement + : singleSelectStatement (setOperator^ singleSelectStatement)* + ; + +singleSelectStatement : selectClause fromClause @@ -1912,7 +1908,6 @@ selectStatement distributeByClause? sortByClause? window_clause? limitClause?)) ; - body : insertClause diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 1ab5a60..6b6b296 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -78,7 +78,7 @@ public boolean existsTable() { } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { try { Tree fromTree = ast.getChild(0); // initialize load path @@ -277,6 +277,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } catch (Exception e) { throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg(), e); } + return ast; } private Task loadTable(URI fromURI, Table table) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 5663fca..c6acf9b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -155,7 +155,7 @@ private void applyConstraints(URI fromURI, URI toURI, Tree ast, } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { isLocal = false; isOverWrite = false; Tree fromTree = ast.getChild(0); @@ -304,5 +304,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { else if (statTask != null) { childTask.addDependentTask(statTask); } + + return ast; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java index b42a425..5be340c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java @@ -59,7 +59,7 @@ public MacroSemanticAnalyzer(HiveConf conf) throws SemanticException { } @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { if (ast.getToken().getType() == HiveParser.TOK_CREATEMACRO) { LOG.debug("Analyzing create macro " + ast.dump()); analyzeCreateMacro(ast); @@ -68,6 +68,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { LOG.debug("Analyzing drop macro " + ast.dump()); analyzeDropMacro(ast); } + return ast; } @SuppressWarnings("unchecked") diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8e68fcf..04bcb51 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -233,6 +233,9 @@ // Max characters when auto generating the column name with func name private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20; + // counter to generate unique union aliases + private int aliasCounter; + // flag for no scan during analyze ... compute statistics protected boolean noscan = false; @@ -336,6 +339,19 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) assert (ast.getToken() != null); switch (ast.getToken().getType()) { + case HiveParser.TOK_DESTINATION: + case HiveParser.TOK_INSERT_INTO: + // there is a insert in the subquery + ASTNode ch = (ASTNode) ast.getChild(0); // TOK_TAB + if (ch != null) { + ch = (ASTNode) ch.getChild(0); // TOK_TABNAME + } + if (ch != null) { + ch = (ASTNode) ch.getChild(0); // name + } + throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY + .getMsg(ch)); + // break; case HiveParser.TOK_QUERY: { QB qb = new QB(id, alias, true); Phase1Ctx ctx_1 = initPhase1Ctx(); @@ -627,6 +643,10 @@ private void assertCombineInputFormat(Tree numerator, String message) throws Sem } } + private String generateUnionAlias() { + return "_u" + (++aliasCounter); + } + private String processSubQuery(QB qb, ASTNode subq) throws SemanticException { // This is a subquery and must have an alias @@ -967,9 +987,6 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) break; case HiveParser.TOK_UNION: - // currently, we dont support subq1 union subq2 - the user has to - // explicitly say: - // select * from (subq1 union subq2) subqalias if (!qbp.getIsSubQ()) { throw new SemanticException(generateErrorMessage(ast, ErrorMsg.UNION_NOTIN_SUBQ.getMsg())); @@ -8852,13 +8869,26 @@ public void init() { @Override @SuppressWarnings("nls") - public void analyzeInternal(ASTNode ast) throws SemanticException { - ASTNode child = ast; - this.ast = ast; + public ASTNode analyzeInternal(ASTNode ast) throws SemanticException { viewsExpanded = new ArrayList(); + aliasCounter = 0; LOG.info("Starting Semantic Analysis"); + if (ast.getToken().getType() == HiveParser.TOK_INSERT_INTO + || ast.getToken().getType() == HiveParser.TOK_DESTINATION) { + // change ast to contain "insert into" in query block + ast = SubQueryUtils.buildTopLevelInsert(ast, generateUnionAlias()); + } + + if (ast.getToken().getType() == HiveParser.TOK_UNION) { + // rewrite as subquery + ast = SubQueryUtils.buildTopLevelUnionQuery(ast, generateUnionAlias()); + } + + ASTNode child = ast; + this.ast = ast; + // analyze and process the position alias processPositionAlias(ast); @@ -8866,7 +8896,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) { // if it is not CTAS, we don't need to go further and just return if ((child = analyzeCreateTable(ast, qb)) == null) { - return; + return this.ast; } } else { SessionState.get().setCommandType(HiveOperation.QUERY); @@ -8878,9 +8908,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { child = analyzeCreateView(ast, qb); SessionState.get().setCommandType(HiveOperation.CREATEVIEW); if (child == null) { - return; + return this.ast; } - viewSelect = child; // prevent view from referencing itself viewsExpanded.add(SessionState.get().getCurrentDatabase() + "." + createVwDesc.getViewName()); } @@ -8889,7 +8918,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { Phase1Ctx ctx_1 = initPhase1Ctx(); if (!doPhase1(child, qb, ctx_1)) { // if phase1Result false return - return; + return this.ast; } LOG.info("Completed phase 1 of Semantic Analysis"); @@ -8919,7 +8948,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // skip the rest of this method. ctx.setResDir(null); ctx.setResFile(null); - return; + return this.ast; } ParseContext pCtx = new ParseContext(conf, qb, child, opToPartPruner, @@ -8968,7 +8997,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { LOG.info("Completed plan generation"); - return; + return this.ast; } @Override @@ -9458,6 +9487,10 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) command_type = CTLT; } break; + case HiveParser.TOK_UNION: // CTAS top level union all + child = SubQueryUtils.buildTopLevelUnionQuery(child, generateUnionAlias()); + ast.replaceChildren(num, num, child); + // fallthrough case HiveParser.TOK_QUERY: // CTAS if (command_type == CTLT) { throw new SemanticException(ErrorMsg.CTAS_CTLT_COEXISTENCE.getMsg()); @@ -9661,7 +9694,14 @@ private ASTNode analyzeCreateView(ASTNode ast, QB qb) case HiveParser.TOK_ORREPLACE: orReplace = true; break; + case HiveParser.TOK_UNION: // CTAS top level union all + viewSelect = child; + child = SubQueryUtils.buildTopLevelUnionQuery(child, generateUnionAlias()); + ast.replaceChildren(num, num, child); + selectStmt = child; + break; case HiveParser.TOK_QUERY: + viewSelect = child; selectStmt = child; break; case HiveParser.TOK_TABCOLNAME: diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java index 8ffbe07..558dfa5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java @@ -9,6 +9,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.parse.QBSubQuery.SubQueryType; import org.apache.hadoop.hive.ql.parse.QBSubQuery.SubQueryTypeDef; @@ -436,6 +437,86 @@ static void setOriginDeep(ASTNode node, ASTNodeOrigin origin) { } } + /** + * buildTopLevelInsert takes a top level insert and rewrites it to what the normal + * shape of queries is. That is it takes the top level insert clause and replaces the + * insert into temp storage of the query with it. + */ + static ASTNode buildTopLevelInsert(ASTNode ast, String alias) throws SemanticException { + int queryIndex = 1; + ASTNode query = (ASTNode) ast.getChild(queryIndex); + + if (query.getToken().getType() == HiveParser.TOK_IFNOTEXISTS) { + queryIndex = 2; + query = (ASTNode) ast.getChild(queryIndex); + } + + if (query.getToken().getType() == HiveParser.TOK_UNION) { + query = buildTopLevelUnionQuery(query, alias); + } + + ASTNode insert = (ASTNode) query.getChild(1); + ASTNode destination = (ASTNode) insert.getChild(0); + + ast.deleteChild(queryIndex); + insert.replaceChildren(0, 0, ast); + + return query; + } + + /** + * buildTopLevelUnionQuery takes a union query + * TOK_UNION + * TOK_QUERY + * ... + * TOK_QUERY + * ... + * and wraps it into a subquery + * TOK_QUERY + * TOK_FROM + * TOK_SUBQUERY + * TOK_UNION + * ... + * TOK_INSERT + * ... + */ + static ASTNode buildTopLevelUnionQuery(ASTNode unionAST, String alias) { + + // create query ast + ASTNode qry = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_QUERY, "TOK_QUERY"); + ASTNode from = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FROM, "TOK_FROM"); + ASTNode sqExpr = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_SUBQUERY, "TOK_SUBQUERY"); + + sqExpr.addChild(unionAST); + sqExpr.addChild(createAliasAST(alias)); + from.addChild(sqExpr); + qry.addChild(from); + + // create insert ast + ASTNode insert = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_INSERT, "TOK_INSERT"); + ASTNode dest = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_DESTINATION, "TOK_DESTINATION"); + ASTNode dir = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_DIR, "TOK_DIR"); + ASTNode tfile = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_TMP_FILE, "TOK_TMP_FILE"); + + insert.addChild(dest); + dest.addChild(dir); + dir.addChild(tfile); + + qry.addChild(insert); + + // create select ast + ASTNode select = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_SELECT, "TOK_SELECT"); + ASTNode selectExpr = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_SELEXPR, "TOK_SELEXPR"); + ASTNode star = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_ALLCOLREF, "TOK_ALLCOLREF"); + + select.addChild(selectExpr); + selectExpr.addChild(star); + + insert.addChild(select); + + return qry; + } + /* * Set of functions to create the Null Check Query for Not-In SubQuery predicates. * For a SubQuery predicate like: diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java index 93e3ad3..f5e73b6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java @@ -218,10 +218,12 @@ void addCopyTranslation(ASTNode targetNode, ASTNode sourceNode) { */ void applyTranslations(TokenRewriteStream tokenRewriteStream) { for (Map.Entry entry : translations.entrySet()) { - tokenRewriteStream.replace( - entry.getKey(), - entry.getValue().tokenStopIndex, - entry.getValue().replacementText); + if (entry.getKey() > 0) { // negative means the key didn't exist in the original stream (i.e.: we changed the tree) + tokenRewriteStream.replace( + entry.getKey(), + entry.getValue().tokenStopIndex, + entry.getValue().replacementText); + } } for (CopyTranslation copyTranslation : copyTranslations) { String replacementText = tokenRewriteStream.toString( diff --git ql/src/test/queries/clientnegative/union.q ql/src/test/queries/clientnegative/union.q deleted file mode 100644 index e3c5c83..0000000 --- ql/src/test/queries/clientnegative/union.q +++ /dev/null @@ -1,4 +0,0 @@ -explain -select s1.key as key, s1.value as value from src s1 - UNION ALL -select s2.key as key, s2.value as value from src s2; diff --git ql/src/test/queries/clientpositive/union_top_level.q ql/src/test/queries/clientpositive/union_top_level.q new file mode 100644 index 0000000..ff9d92c --- /dev/null +++ ql/src/test/queries/clientpositive/union_top_level.q @@ -0,0 +1,98 @@ +-- top level +explain +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +explain +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10; + +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10; + +-- ctas +explain +create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +select * from union_top; + +truncate table union_top; + +-- insert into +explain +insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +select * from union_top; + +explain +insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +select * from union_top; + +-- create view +explain +create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3; + +select * from union_top_view; + +drop table union_top; +drop view union_top_view; diff --git ql/src/test/results/clientnegative/union.q.out ql/src/test/results/clientnegative/union.q.out deleted file mode 100644 index b66d394..0000000 --- ql/src/test/results/clientnegative/union.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException 2:45 Top level UNION is not supported currently; use a subquery for the UNION. Error encountered near token 'value' diff --git ql/src/test/results/clientpositive/union_top_level.q.out ql/src/test/results/clientpositive/union_top_level.q.out new file mode 100644 index 0000000..6c21b15 --- /dev/null +++ ql/src/test/results/clientpositive/union_top_level.q.out @@ -0,0 +1,1506 @@ +PREHOOK: query: -- top level +explain +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +POSTHOOK: query: -- top level +explain +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 0 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 0)) (TOK_LIMIT 3))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 1)) (TOK_LIMIT 3)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 2)) (TOK_LIMIT 3)))) _u1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery1:_u1-subquery1-subquery1:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 0 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:_u1-subquery1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 1) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 1 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:_u1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 2) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 2 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 1 +409 1 +484 1 +27 0 +165 0 +255 0 +86 2 +311 2 +278 2 +PREHOOK: query: explain +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) s1) (TOK_TABREF (TOK_TABNAME src) s2) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s1) key) k) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) value) v)) (TOK_LIMIT 10))) (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) s1) (TOK_TABREF (TOK_TABNAME src) s2) (= (. (TOK_TABLE_OR_COL s1) key) (. (TOK_TABLE_OR_COL s2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s1) key) k) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) value) v)) (TOK_LIMIT 10)))) _u1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-5 is a root stage + Stage-6 depends on stages: Stage-5 + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:_u1-subquery1:s1 + TableScan + alias: s1 + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 0 + value expressions: + expr: key + type: string + null-subquery1:_u1-subquery1:s2 + TableScan + alias: s2 + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {VALUE._col1} + handleSkewJoin: false + outputColumnNames: _col0, _col5 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col5 + type: string + outputColumnNames: _col0, _col1 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:_u1-subquery2:s1 + TableScan + alias: s1 + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 0 + value expressions: + expr: key + type: string + null-subquery2:_u1-subquery2:s2 + TableScan + alias: s2 + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: value + type: string + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {VALUE._col1} + handleSkewJoin: false + outputColumnNames: _col0, _col5 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col5 + type: string + outputColumnNames: _col0, _col1 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +union all +select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +PREHOOK: query: -- ctas +explain +create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: -- ctas +explain +create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: CREATETABLE_AS_SELECT +ABSTRACT SYNTAX TREE: + (TOK_CREATETABLE (TOK_TABNAME union_top) TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 0 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 0)) (TOK_LIMIT 3))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 1)) (TOK_LIMIT 3)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 2)) (TOK_LIMIT 3)))) _u1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 + Stage-5 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-11 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-11 + Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery1:_u1-subquery1-subquery1:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 0 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-8 + Conditional Operator + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-11 + Create Table Operator: + Create Table + columns: key string, value int + if not exists: false + input format: org.apache.hadoop.mapred.TextInputFormat + # buckets: -1 + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + name: union_top + isExternal: false + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:_u1-subquery1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 1) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 1 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:_u1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 2) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 2 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +POSTHOOK: query: create table union_top as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_top +PREHOOK: query: select * from union_top +PREHOOK: type: QUERY +PREHOOK: Input: default@union_top +#### A masked pattern was here #### +POSTHOOK: query: select * from union_top +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_top +#### A masked pattern was here #### +238 1 +409 1 +484 1 +27 0 +165 0 +255 0 +86 2 +311 2 +278 2 +PREHOOK: query: truncate table union_top +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@union_top +POSTHOOK: query: truncate table union_top +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@union_top +PREHOOK: query: -- insert into +explain +insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +POSTHOOK: query: -- insert into +explain +insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 0 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 0)) (TOK_LIMIT 3))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 1)) (TOK_LIMIT 3)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 2)) (TOK_LIMIT 3)))) _u1)) (TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME union_top))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 + Stage-5 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-3 depends on stages: Stage-0 + Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery1:_u1-subquery1-subquery1:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 0 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-8 + Conditional Operator + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:_u1-subquery1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 1) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 1 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:_u1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 2) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 2 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_top +POSTHOOK: query: insert into table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_top +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +PREHOOK: query: select * from union_top +PREHOOK: type: QUERY +PREHOOK: Input: default@union_top +#### A masked pattern was here #### +POSTHOOK: query: select * from union_top +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_top +#### A masked pattern was here #### +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +238 1 +409 1 +484 1 +27 0 +165 0 +255 0 +86 2 +311 2 +278 2 +PREHOOK: query: explain +insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 0 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 0)) (TOK_LIMIT 3))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 1)) (TOK_LIMIT 3)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 2)) (TOK_LIMIT 3)))) _u1)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME union_top))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 + Stage-5 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-3 depends on stages: Stage-0 + Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 + Stage-9 is a root stage + Stage-10 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery1:_u1-subquery1-subquery1:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 0) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 0 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top +#### A masked pattern was here #### + TableScan + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: int + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-8 + Conditional Operator + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-6 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + TableScan + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_top + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:_u1-subquery1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 1) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 1 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:_u1-subquery2:src + TableScan + alias: src + Filter Operator + predicate: + expr: ((key % 3) = 2) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: 2 + type: int + outputColumnNames: _col0, _col1 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_top +POSTHOOK: query: insert overwrite table union_top +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_top +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +PREHOOK: query: select * from union_top +PREHOOK: type: QUERY +PREHOOK: Input: default@union_top +#### A masked pattern was here #### +POSTHOOK: query: select * from union_top +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_top +#### A masked pattern was here #### +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +238 1 +409 1 +484 1 +27 0 +165 0 +255 0 +86 2 +311 2 +278 2 +PREHOOK: query: -- create view +explain +create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: -- create view +explain +create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +ABSTRACT SYNTAX TREE: + (TOK_CREATEVIEW (TOK_TABNAME union_top_view) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 0 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 0)) (TOK_LIMIT 3))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 1)) (TOK_LIMIT 3)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 value)) (TOK_WHERE (== (% (TOK_TABLE_OR_COL key) 3) 2)) (TOK_LIMIT 3)))) _u1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create View Operator: + Create View + if not exists: false + is alter view as select: false + or replace: false + columns: key string, value int + expanded text: select `src`.`key`, 0 as `value` from `default`.`src` where `src`.`key` % 3 == 0 limit 3 +union all +select `src`.`key`, 1 as `value` from `default`.`src` where `src`.`key` % 3 == 1 limit 3 +union all +select `src`.`key`, 2 as `value` from `default`.`src` where `src`.`key` % 3 == 2 limit 3 + name: union_top_view + original text: select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 + +PREHOOK: query: create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: create view union_top_view as +select key, 0 as value from src where key % 3 == 0 limit 3 +union all +select key, 1 as value from src where key % 3 == 1 limit 3 +union all +select key, 2 as value from src where key % 3 == 2 limit 3 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@union_top_view +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +PREHOOK: query: select * from union_top_view +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@union_top_view +#### A masked pattern was here #### +POSTHOOK: query: select * from union_top_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@union_top_view +#### A masked pattern was here #### +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +238 1 +409 1 +484 1 +27 0 +165 0 +255 0 +86 2 +311 2 +278 2 +PREHOOK: query: drop table union_top +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_top +PREHOOK: Output: default@union_top +POSTHOOK: query: drop table union_top +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_top +POSTHOOK: Output: default@union_top +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +PREHOOK: query: drop view union_top_view +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@union_top_view +PREHOOK: Output: default@union_top_view +POSTHOOK: query: drop view union_top_view +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@union_top_view +POSTHOOK: Output: default@union_top_view +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_top.value EXPRESSION [] +POSTHOOK: Lineage: union_top.value EXPRESSION [] diff --git ql/src/test/results/compiler/parse/sample2.q.out ql/src/test/results/compiler/parse/sample2.q.out index e67c761..d3911c1 100644 --- ql/src/test/results/compiler/parse/sample2.q.out +++ ql/src/test/results/compiler/parse/sample2.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s))))))) \ No newline at end of file diff --git ql/src/test/results/compiler/parse/sample3.q.out ql/src/test/results/compiler/parse/sample3.q.out index ad5855b..0de7f80 100644 --- ql/src/test/results/compiler/parse/sample3.q.out +++ ql/src/test/results/compiler/parse/sample3.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2 (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2 (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s))))))) \ No newline at end of file diff --git ql/src/test/results/compiler/parse/sample4.q.out ql/src/test/results/compiler/parse/sample4.q.out index 790b009..58cd9ca 100644 --- ql/src/test/results/compiler/parse/sample4.q.out +++ ql/src/test/results/compiler/parse/sample4.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 2 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s))))))) \ No newline at end of file diff --git ql/src/test/results/compiler/parse/sample5.q.out ql/src/test/results/compiler/parse/sample5.q.out index cb55074..8247cc8 100644 --- ql/src/test/results/compiler/parse/sample5.q.out +++ ql/src/test/results/compiler/parse/sample5.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 5 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 5 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s))))))) \ No newline at end of file diff --git ql/src/test/results/compiler/parse/sample6.q.out ql/src/test/results/compiler/parse/sample6.q.out index 3562bb8..9dd4b72 100644 --- ql/src/test/results/compiler/parse/sample6.q.out +++ ql/src/test/results/compiler/parse/sample6.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 4 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 4 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s))))))) \ No newline at end of file diff --git ql/src/test/results/compiler/parse/sample7.q.out ql/src/test/results/compiler/parse/sample7.q.out index 6bcf840..f883f99 100644 --- ql/src/test/results/compiler/parse/sample7.q.out +++ ql/src/test/results/compiler/parse/sample7.q.out @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 4 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL s) key) 100)))) \ No newline at end of file +(TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1)) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcbucket) (TOK_TABLEBUCKETSAMPLE 1 4 (TOK_TABLE_OR_COL key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME s)))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL s) key) 100))))) \ No newline at end of file