diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 17f3552..8aae1ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -317,6 +317,7 @@ TOK_SUBQUERY_OP_NOTIN; TOK_SUBQUERY_OP_NOTEXISTS; TOK_DB_TYPE; TOK_TABLE_TYPE; +TOK_CTE; } @@ -787,7 +788,7 @@ createTableStatement tableFileFormat? tableLocation? tablePropertiesPrefixed? - (KW_AS selectStatement[true])? + (KW_AS selectStatementWithCTE)? ) -> ^(TOK_CREATETABLE $name $ext? ifNotExists? ^(TOK_LIKETABLE $likeName?) @@ -800,7 +801,7 @@ createTableStatement tableFileFormat? tableLocation? tablePropertiesPrefixed? - selectStatement? + selectStatementWithCTE? ) ; @@ -939,8 +940,8 @@ alterViewStatementSuffix -> ^(TOK_ALTERVIEW_ADDPARTS alterStatementSuffixAddPartitions) | alterStatementSuffixDropPartitions -> ^(TOK_ALTERVIEW_DROPPARTS alterStatementSuffixDropPartitions) - | name=tableName KW_AS selectStatement[true] - -> ^(TOK_ALTERVIEW_AS $name selectStatement) + | name=tableName KW_AS selectStatementWithCTE + -> ^(TOK_ALTERVIEW_AS $name selectStatementWithCTE) ; alterIndexStatementSuffix @@ -1511,14 +1512,14 @@ createViewStatement (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition? tablePropertiesPrefixed? KW_AS - selectStatement[true] + selectStatementWithCTE -> ^(TOK_CREATEVIEW $name orReplace? ifNotExists? columnNameCommentList? tableComment? viewPartition? tablePropertiesPrefixed? - selectStatement + selectStatementWithCTE ) ; @@ -1910,10 +1911,36 @@ setOperator queryStatementExpression[boolean topLevel] : + /* Would be nice to do this as a gated semantic perdicate + But the predicate gets pushed as a lookahead decision. + Calling rule doesnot know about topLevel + */ + (w=withClause {topLevel}?)? + queryStatementExpressionBody[topLevel] { + if ($w.tree != null) { + adaptor.addChild($queryStatementExpressionBody.tree, $w.tree); + } + } + -> queryStatementExpressionBody + ; + +queryStatementExpressionBody[boolean topLevel] + : fromStatement[topLevel] | regularBody[topLevel] ; - + +withClause + : + KW_WITH cteStatement (COMMA cteStatement)* -> ^(TOK_CTE cteStatement+) +; + +cteStatement + : + identifier KW_AS LPAREN queryStatementExpression[false] RPAREN + -> ^(TOK_SUBQUERY queryStatementExpression identifier) +; + fromStatement[boolean topLevel] : (singleFromStatement -> singleFromStatement) (u=setOperator r=singleFromStatement @@ -1987,6 +2014,17 @@ singleSelectStatement distributeByClause? sortByClause? window_clause? limitClause?)) ; +selectStatementWithCTE + : + (w=withClause)? + selectStatement[true] { + if ($w.tree != null) { + adaptor.addChild($selectStatement.tree, $w.tree); + } + } + -> selectStatement + ; + body : insertClause diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 3fbe8e2..a8b436e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -219,6 +219,10 @@ public void rewriteViewToSubq(String alias, String viewName, QBExpr qbexpr) { aliasToSubq.put(alias, qbexpr); } + public void rewriteCTEToSubq(String alias, String cteName, QBExpr qbexpr) { + rewriteViewToSubq(alias, cteName, qbexpr); + } + public QBJoinTree getQbJoinTree() { return qbjoin; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 59987e3..b2c7046 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -241,6 +241,15 @@ //flag for partial scan during analyze ... compute statistics protected boolean partialscan = false; + /* + * Capture the CTE definitions in a Query. + */ + private final Map aliasToCTEs; + /* + * Used to check recursive CTE invocations. Similar to viewsExpanded + */ + private ArrayList ctesExpanded; + private static class Phase1Ctx { String dest; int nextNum; @@ -276,6 +285,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME); queryProperties = new QueryProperties(); opToPartToSkewedPruner = new HashMap>(); + aliasToCTEs = new HashMap(); } @Override @@ -295,6 +305,7 @@ protected void reset() { opParseCtx.clear(); groupOpToInputTables.clear(); prunedPartitions.clear(); + aliasToCTEs.clear(); } public void initParseCtx(ParseContext pctx) { @@ -657,6 +668,98 @@ private String processSubQuery(QB qb, ASTNode subq) throws SemanticException { return alias; } + /* + * Phase1: hold onto any CTE definitions in aliasToCTE. + * CTE definitions are global to the Query. + */ + private void processCTE(QB qb, ASTNode ctes) throws SemanticException { + + int numCTEs = ctes.getChildCount(); + + for(int i=0; i aliasToCTEs map. + * + */ + private ASTNode findCTEFromName(QB qb, String cteName) { + + /* + * When saving a view definition all table references in the AST are qualified; including CTE references. + * Where as CTE definitions have no DB qualifier; so we strip out the DB qualifier before searching in + * aliasToCTEs map. + */ + String currDB = SessionState.get().getCurrentDatabase(); + if ( currDB != null && cteName.startsWith(currDB) && + cteName.length() > currDB.length() && + cteName.charAt(currDB.length()) == '.' ) { + cteName = cteName.substring(currDB.length() + 1); + } + + StringBuffer qId = new StringBuffer(); + if (qb.getId() != null) { + qId.append(qb.getId()); + } + + while (qId.length() > 0) { + String nm = qId + ":" + cteName; + if (aliasToCTEs.containsKey(nm)) { + return aliasToCTEs.get(nm); + } + int lastIndex = qId.lastIndexOf(":"); + lastIndex = lastIndex < 0 ? 0 : lastIndex; + qId.setLength(lastIndex); + } + return aliasToCTEs.get(cteName); + } + + /* + * If a CTE is referenced in a QueryBlock: + * - add it as a SubQuery for now. + * - SQ.alias is the alias used in QB. (if no alias is specified, + * it used the CTE name. Works just like table references) + * - Adding SQ done by: + * - copying AST of CTE + * - setting ASTOrigin on cloned AST. + * - trigger phase 1 on new QBExpr. + * - update QB data structs: remove this as a table reference, move it to a SQ invocation. + */ + private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) throws SemanticException { + cteAlias = cteAlias == null ? cteName : cteAlias; + ASTNode cteQryNode = findCTEFromName(qb, cteName); + QBExpr cteQBExpr = new QBExpr(cteAlias); + + String cteText = ctx.getTokenRewriteStream().toString( + cteQryNode.getTokenStartIndex(), cteQryNode.getTokenStopIndex()); + final ASTNodeOrigin cteOrigin = new ASTNodeOrigin("CTE", cteName, + cteText, cteAlias, cteQryNode); + cteQryNode = (ASTNode) ParseDriver.adaptor.dupTree(cteQryNode); + SubQueryUtils.setOriginDeep(cteQryNode, cteOrigin); + + doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias); + qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr); + } + private boolean isJoinToken(ASTNode node) { if ((node.getToken().getType() == HiveParser.TOK_JOIN) || (node.getToken().getType() == HiveParser.TOK_CROSSJOIN) @@ -1032,6 +1135,9 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) assert ast.getChildCount() == 1; qb.getParseInfo().getDestToLateralView().put(ctx_1.dest, ast); break; + case HiveParser.TOK_CTE: + processCTE(qb, ast); + break; default: skipRecursion = false; break; @@ -1084,12 +1190,35 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException // This is needed for tracking the dependencies for inputs, along with their parents. Map> aliasToViewInfo = new HashMap>(); + + /* + * used to capture view to SQ conversions. This is used to check for + * recursive CTE invocations. + */ + Map sqAliasToCTEName = new HashMap(); + for (String alias : tabAliases) { String tab_name = qb.getTabNameForAlias(alias); Table tab = null; try { tab = db.getTable(tab_name); } catch (InvalidTableException ite) { + /* + * if this s a CTE reference: + * Add its AST as a SubQuery to this QB. + */ + ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase()); + if ( cteNode != null ) { + String cte_name = tab_name.toLowerCase(); + if (ctesExpanded.contains(cte_name)) { + throw new SemanticException("Recursive cte " + tab_name + + " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") + + " -> " + tab_name + ")."); + } + addCTEAsSubQuery(qb, cte_name, alias); + sqAliasToCTEName.put(alias, cte_name); + continue; + } throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(qb .getParseInfo().getSrcForAlias(alias))); } @@ -1182,15 +1311,20 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException // Go over the subqueries and getMetaData for these for (String alias : qb.getSubqAliases()) { boolean wasView = aliasToViewInfo.containsKey(alias); + boolean wasCTE = sqAliasToCTEName.containsKey(alias); ReadEntity newParentInput = null; if (wasView) { viewsExpanded.add(aliasToViewInfo.get(alias).getFirst()); newParentInput = aliasToViewInfo.get(alias).getSecond(); + } else if (wasCTE) { + ctesExpanded.add(sqAliasToCTEName.get(alias)); } QBExpr qbexpr = qb.getSubqForAlias(alias); getMetaData(qbexpr, newParentInput); if (wasView) { viewsExpanded.remove(viewsExpanded.size() - 1); + } else if (wasCTE) { + ctesExpanded.remove(ctesExpanded.size() - 1); } } @@ -8867,6 +9001,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { ASTNode child = ast; this.ast = ast; viewsExpanded = new ArrayList(); + ctesExpanded = new ArrayList(); LOG.info("Starting Semantic Analysis"); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java index 4ac8f07..e91258a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java @@ -322,7 +322,7 @@ private PrivilegeObjectDesc analyzePrivilegeObject(ASTNode ast, Privilege privObj = PrivilegeRegistry.getPrivilege(privilegeType.getType()); if (privObj == null) { - throw new SemanticException("undefined privilege " + privilegeType.getType()); + throw new SemanticException("undefined privilege " + privilegeType.getText()); } List cols = null; if (privilegeDef.getChildCount() > 1) { diff --git ql/src/test/queries/clientnegative/cte_recursion.q ql/src/test/queries/clientnegative/cte_recursion.q new file mode 100644 index 0000000..2160b47 --- /dev/null +++ ql/src/test/queries/clientnegative/cte_recursion.q @@ -0,0 +1,4 @@ +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from q1 where key = '5') +select * from (select key from q1) a; \ No newline at end of file diff --git ql/src/test/queries/clientnegative/cte_with_in_subquery.q ql/src/test/queries/clientnegative/cte_with_in_subquery.q new file mode 100644 index 0000000..e52a1d9 --- /dev/null +++ ql/src/test/queries/clientnegative/cte_with_in_subquery.q @@ -0,0 +1 @@ +select * from (with q1 as ( select key from q2 where key = '5') select * from q1) a; diff --git ql/src/test/queries/clientpositive/cte_1.q ql/src/test/queries/clientpositive/cte_1.q new file mode 100644 index 0000000..ca4132f --- /dev/null +++ ql/src/test/queries/clientpositive/cte_1.q @@ -0,0 +1,28 @@ +explain +with q1 as ( select key from src where key = '5') +select * +from q1 +; + +with q1 as ( select key from src where key = '5') +select * +from q1 +; + +-- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a; + +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a; + +-- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a; + +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/cte_2.q ql/src/test/queries/clientpositive/cte_2.q new file mode 100644 index 0000000..b49620a --- /dev/null +++ ql/src/test/queries/clientpositive/cte_2.q @@ -0,0 +1,56 @@ + +-- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +; + +-- insert test +create table s1 like src; +with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +; +select * from s1; +drop table s1; + +-- from style +with q1 as (select * from src where key= '5') +from q1 +select * +; + +-- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +; + +select * from s2; +drop table s2; + +-- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +; + +select * from v1; + +drop view v1; + + +-- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +; + +with q1 as ( select key from src where key = '4') +select * from v1 +; + +drop view v1; + + diff --git ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out index 431f16e..425745e 100644 --- ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out +++ ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table if not exists authorization_invalid_v1 (key int, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@authorization_invalid_v1 -FAILED: SemanticException undefined privilege 731 +FAILED: SemanticException undefined privilege TOK_PRIV_DELETE diff --git ql/src/test/results/clientnegative/cte_recursion.q.out ql/src/test/results/clientnegative/cte_recursion.q.out new file mode 100644 index 0000000..10aed92 --- /dev/null +++ ql/src/test/results/clientnegative/cte_recursion.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Recursive cte q1 detected (cycle: q1 -> q2 -> q1). diff --git ql/src/test/results/clientnegative/cte_with_in_subquery.q.out ql/src/test/results/clientnegative/cte_with_in_subquery.q.out new file mode 100644 index 0000000..fa22b48 --- /dev/null +++ ql/src/test/results/clientnegative/cte_with_in_subquery.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 1:64 Failed to recognize predicate 'select'. Failed rule: 'queryStatementExpression' in subquery source diff --git ql/src/test/results/clientpositive/cte_1.q.out ql/src/test/results/clientpositive/cte_1.q.out new file mode 100644 index 0000000..e862975 --- /dev/null +++ ql/src/test/results/clientpositive/cte_1.q.out @@ -0,0 +1,165 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/cte_2.q.out ql/src/test/results/clientpositive/cte_2.q.out new file mode 100644 index 0000000..f4573b3 --- /dev/null +++ ql/src/test/results/clientpositive/cte_2.q.out @@ -0,0 +1,201 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 val_4 +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@s1 +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@s1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: default@s2 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: default@v1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: default@v1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]