diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 43c2d56..a03263e 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -165,6 +165,8 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ union7.q,\ union8.q,\ union9.q,\ + unionDistinct_1.q,\ + unionDistinct_2.q,\ update_after_multiple_inserts.q,\ update_all_non_partitioned.q,\ update_all_partitioned.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index 95ad9e0..ea59181 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -722,7 +722,7 @@ public ASTNode getAST() { public ASTNode getUnionAllAST(ASTNode leftAST, ASTNode rightAST) { - ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNION, "TOK_UNION").add(leftAST) + ASTNode unionTokAST = ASTBuilder.construct(HiveParser.TOK_UNIONALL, "TOK_UNIONALL").add(leftAST) .add(rightAST).node(); return unionTokAST; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 9c7603c..fca79e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -85,7 +85,8 @@ TOK_ORDERBY; TOK_CLUSTERBY; TOK_DISTRIBUTEBY; TOK_SORTBY; -TOK_UNION; +TOK_UNIONALL; +TOK_UNIONDISTINCT; TOK_JOIN; TOK_LEFTOUTERJOIN; TOK_RIGHTOUTERJOIN; @@ -616,6 +617,10 @@ import java.util.HashMap; private String generateUnionAlias() { return "_u" + (++aliasCounter); } + + private CommonTree throwSetOpException() throws RecognitionException { + throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.", ""); + } } @rulecatch { @@ -2043,7 +2048,8 @@ unionType setOperator @init { pushMsg("set operator", state); } @after { popMsg(state); } - : KW_UNION KW_ALL -> ^(TOK_UNION) + : KW_UNION KW_ALL -> ^(TOK_UNIONALL) + | KW_UNION KW_DISTINCT? -> ^(TOK_UNIONDISTINCT) ; queryStatementExpression[boolean topLevel] @@ -2131,41 +2137,100 @@ regularBody[boolean topLevel] selectStatement[topLevel] ; - selectStatement[boolean topLevel] - : (singleSelectStatement -> singleSelectStatement) - (u=setOperator b=singleSelectStatement - -> ^($u {$selectStatement.tree} $b) - )* - -> {u != null && topLevel}? ^(TOK_QUERY - ^(TOK_FROM - ^(TOK_SUBQUERY - {$selectStatement.tree} +selectStatement[boolean topLevel] + : + ( + s=selectClause + f=fromClause? + w=whereClause? + g=groupByClause? + h=havingClause? + o=orderByClause? + c=clusterByClause? + d=distributeByClause? + sort=sortByClause? + win=window_clause? + l=limitClause? + -> ^(TOK_QUERY $f? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + $s $w? $g? $h? $o? $c? + $d? $sort? $win? $l?)) + ) + (set=setOpSelectStatement[$selectStatement.tree, topLevel]?) + -> {set == null}? + ^(TOK_QUERY $f? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + $s $w? $g? $h? $o? $c? + $d? $sort? $win? $l?)) + -> {o==null && c==null && d==null && sort==null && l==null}? + {$set.tree} + -> {throwSetOpException()} + ; + +setOpSelectStatement[CommonTree t, boolean topLevel] + : + (u=setOperator b=simpleSelectStatement + -> {$setOpSelectStatement.tree != null && u.tree.getType()==HiveParser.TOK_UNIONDISTINCT}? + ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_SUBQUERY + ^(TOK_UNIONALL {$setOpSelectStatement.tree} $b) {adaptor.create(Identifier, generateUnionAlias())} ) ) - ^(TOK_INSERT + ^(TOK_INSERT + ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + ^(TOK_SELECTDI ^(TOK_SELEXPR TOK_ALLCOLREF)) + ) + ) + -> {$setOpSelectStatement.tree != null && u.tree.getType()!=HiveParser.TOK_UNIONDISTINCT}? + ^(TOK_UNIONALL {$setOpSelectStatement.tree} $b) + -> {$setOpSelectStatement.tree == null && u.tree.getType()==HiveParser.TOK_UNIONDISTINCT}? + ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_SUBQUERY + ^(TOK_UNIONALL {$t} $b) + {adaptor.create(Identifier, generateUnionAlias())} + ) + ) + ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) - ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF)) + ^(TOK_SELECTDI ^(TOK_SELEXPR TOK_ALLCOLREF)) + ) + ) + -> ^(TOK_UNIONALL {$t} $b) + )+ + o=orderByClause? + c=clusterByClause? + d=distributeByClause? + sort=sortByClause? + win=window_clause? + l=limitClause? + -> {o==null && c==null && d==null && sort==null && win==null && l==null && !topLevel}? + {$setOpSelectStatement.tree} + -> ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_SUBQUERY + {$setOpSelectStatement.tree} + {adaptor.create(Identifier, generateUnionAlias())} + ) ) - ) - -> {$selectStatement.tree} - ; + ^(TOK_INSERT + ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF)) + $o? $c? $d? $sort? $win? $l? + ) + ) + ; -singleSelectStatement +simpleSelectStatement : selectClause fromClause? whereClause? groupByClause? havingClause? - orderByClause? - clusterByClause? - distributeByClause? - sortByClause? window_clause? - limitClause? -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) - selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause? - distributeByClause? sortByClause? window_clause? limitClause?)) + -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + selectClause whereClause? groupByClause? havingClause? window_clause?)) ; selectStatementWithCTE diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index c960a6b..ce91406 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -553,5 +553,5 @@ principalIdentifier nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER + KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5ff9552..f5e3ab7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -423,7 +423,7 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) qbexpr.setQB(qb); } break; - case HiveParser.TOK_UNION: { + case HiveParser.TOK_UNIONALL: { qbexpr.setOpcode(QBExpr.Opcode.UNION); // query 1 assert (ast.getChild(0) != null); @@ -433,7 +433,7 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) qbexpr.setQBExpr1(qbexpr1); // query 2 - assert (ast.getChild(0) != null); + assert (ast.getChild(1) != null); QBExpr qbexpr2 = new QBExpr(alias + "-subquery2"); doPhase1QBExpr((ASTNode) ast.getChild(1), qbexpr2, id + "-subquery2", alias + "-subquery2"); @@ -1327,7 +1327,7 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan break; - case HiveParser.TOK_UNION: + case HiveParser.TOK_UNIONALL: if (!qbp.getIsSubQ()) { // this shouldn't happen. The parser should have converted the union to be // contained in a subquery. Just in case, we keep the error as a fallback. diff --git a/ql/src/test/queries/clientnegative/unionClusterBy.q b/ql/src/test/queries/clientnegative/unionClusterBy.q new file mode 100644 index 0000000..f2bf837 --- /dev/null +++ b/ql/src/test/queries/clientnegative/unionClusterBy.q @@ -0,0 +1,9 @@ +-- orderByClause clusterByClause distributeByClause sortByClause limitClause +-- can only be applied to the whole union. + +select key from src cluster by key +union all +select key from src; + + + diff --git a/ql/src/test/queries/clientnegative/unionDistributeBy.q b/ql/src/test/queries/clientnegative/unionDistributeBy.q new file mode 100644 index 0000000..fe2a544 --- /dev/null +++ b/ql/src/test/queries/clientnegative/unionDistributeBy.q @@ -0,0 +1,10 @@ +-- orderByClause clusterByClause distributeByClause sortByClause limitClause +-- can only be applied to the whole union. + +select key from src distribute by key +union all +select key from src; + + + + diff --git a/ql/src/test/queries/clientnegative/unionLimit.q b/ql/src/test/queries/clientnegative/unionLimit.q new file mode 100644 index 0000000..91f2af5 --- /dev/null +++ b/ql/src/test/queries/clientnegative/unionLimit.q @@ -0,0 +1,10 @@ +-- orderByClause clusterByClause distributeByClause sortByClause limitClause +-- can only be applied to the whole union. + +select key from src limit 1 +union all +select key from src; + + + + diff --git a/ql/src/test/queries/clientnegative/unionOrderBy.q b/ql/src/test/queries/clientnegative/unionOrderBy.q new file mode 100644 index 0000000..0477e2d --- /dev/null +++ b/ql/src/test/queries/clientnegative/unionOrderBy.q @@ -0,0 +1,9 @@ +-- orderByClause clusterByClause distributeByClause sortByClause limitClause +-- can only be applied to the whole union. + +select key from src order by key +union all +select key from src; + + + diff --git a/ql/src/test/queries/clientnegative/unionSortBy.q b/ql/src/test/queries/clientnegative/unionSortBy.q new file mode 100644 index 0000000..8cf8b90 --- /dev/null +++ b/ql/src/test/queries/clientnegative/unionSortBy.q @@ -0,0 +1,8 @@ +-- orderByClause clusterByClause distributeByClause sortByClause limitClause +-- can only be applied to the whole union. + +select key from src sort by key +union all +select key from src; + + diff --git a/ql/src/test/queries/clientpositive/cbo_union.q b/ql/src/test/queries/clientpositive/cbo_union.q index e9508c5..312d7be 100644 --- a/ql/src/test/queries/clientpositive/cbo_union.q +++ b/ql/src/test/queries/clientpositive/cbo_union.q @@ -7,7 +7,7 @@ set hive.auto.convert.join=false; -- SORT_QUERY_RESULTS -- 11. Union All -select * from cbo_t1 order by key, c_boolean, value, dt union all select * from cbo_t2 order by key, c_boolean, value, dt; +select * from (select * from cbo_t1 order by key, c_boolean, value, dt)a union all select * from (select * from cbo_t2 order by key, c_boolean, value, dt)b; select key from (select key, c_int from (select * from cbo_t1 union all select * from cbo_t2 where cbo_t2.key >=0)r1 union all select key, c_int from cbo_t3)r2 where key >=0 order by key; select r2.key from (select key, c_int from (select key, c_int from cbo_t1 union all select key, c_int from cbo_t3 )r1 union all select key, c_int from cbo_t3)r2 join (select key, c_int from (select * from cbo_t1 union all select * from cbo_t2 where cbo_t2.key >=0)r1 union all select key, c_int from cbo_t3)r3 on r2.key=r3.key where r3.key >=0 order by r2.key; diff --git a/ql/src/test/queries/clientpositive/explode_null.q b/ql/src/test/queries/clientpositive/explode_null.q index 76e4535..6a540f7 100644 --- a/ql/src/test/queries/clientpositive/explode_null.q +++ b/ql/src/test/queries/clientpositive/explode_null.q @@ -1,10 +1,10 @@ SELECT explode(col) AS myCol FROM - (SELECT array(1,2,3) AS col FROM src LIMIT 1 + (select * from (SELECT array(1,2,3) AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1) a; + select * from (SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1)eb) a; SELECT explode(col) AS (myCol1,myCol2) FROM - (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1 + (select * from (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1) a; + select * from (SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1)eb ) a; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/input25.q b/ql/src/test/queries/clientpositive/input25.q index e48368f..202bea2 100644 --- a/ql/src/test/queries/clientpositive/input25.q +++ b/ql/src/test/queries/clientpositive/input25.q @@ -5,15 +5,15 @@ alter table tst add partition (d='2009-02-02'); explain select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq; select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq; diff --git a/ql/src/test/queries/clientpositive/input26.q b/ql/src/test/queries/clientpositive/input26.q index 642a7db..81af75a 100644 --- a/ql/src/test/queries/clientpositive/input26.q +++ b/ql/src/test/queries/clientpositive/input26.q @@ -1,12 +1,12 @@ explain select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq; select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq; diff --git a/ql/src/test/queries/clientpositive/load_dyn_part14.q b/ql/src/test/queries/clientpositive/load_dyn_part14.q index c34c3bf..6c19338 100644 --- a/ql/src/test/queries/clientpositive/load_dyn_part14.q +++ b/ql/src/test/queries/clientpositive/load_dyn_part14.q @@ -14,20 +14,20 @@ set hive.exec.dynamic.partition.mode=nonstrict; explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; diff --git a/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q b/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q index a26ef1a..d530f6a 100644 --- a/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q +++ b/ql/src/test/queries/clientpositive/metadataOnlyOptimizer.q @@ -33,9 +33,9 @@ UNION all SELECT key as key from src ) tab group by key; -select k from (SELECT '1' as k from src limit 0 union all select key as k from src limit 1)tab; +select k from (select * from (SELECT '1' as k from src limit 0)a union all select key as k from src limit 1)tab; -select k from (SELECT '1' as k from src limit 1 union all select key as k from src limit 0)tab; +select k from (select * from (SELECT '1' as k from src limit 1)a union all select key as k from src limit 0)tab; select max(ds) from srcpart; diff --git a/ql/src/test/queries/clientpositive/script_env_var1.q b/ql/src/test/queries/clientpositive/script_env_var1.q index 381c5dc..0148905 100644 --- a/ql/src/test/queries/clientpositive/script_env_var1.q +++ b/ql/src/test/queries/clientpositive/script_env_var1.q @@ -1,5 +1,5 @@ -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key; +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key; diff --git a/ql/src/test/queries/clientpositive/script_env_var2.q b/ql/src/test/queries/clientpositive/script_env_var2.q index 5f10812..01e399b 100644 --- a/ql/src/test/queries/clientpositive/script_env_var2.q +++ b/ql/src/test/queries/clientpositive/script_env_var2.q @@ -1,5 +1,5 @@ set hive.script.operator.id.env.var = MY_ID; -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key; +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key; diff --git a/ql/src/test/queries/clientpositive/union3.q b/ql/src/test/queries/clientpositive/union3.q index d402cb0..0284af0 100644 --- a/ql/src/test/queries/clientpositive/union3.q +++ b/ql/src/test/queries/clientpositive/union3.q @@ -5,17 +5,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a; @@ -27,17 +26,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a; -select * from union_out cluster by id; +select * from union_out; diff --git a/ql/src/test/queries/clientpositive/unionDistinct_1.q b/ql/src/test/queries/clientpositive/unionDistinct_1.q new file mode 100644 index 0000000..61917d7 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionDistinct_1.q @@ -0,0 +1,1134 @@ +-- union10.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + +create table tmptable(key string, value int); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc; + + +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc; + + +select * from tmptable x sort by x.key; + +-- union11.q + + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + + + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + + + +-- union12.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink + +create table tmptable12(key string, value int); + +explain +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc; + + +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc; + +select * from tmptable12 x sort by x.key; +-- union13.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are a map-only jobs, same input, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; +-- union14.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; + + + + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; +-- union15.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + + +-- union16.q + +-- SORT_BEFORE_DIFF +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; + + +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; +-- union17.q + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE; + +-- SORT_BEFORE_DIFF +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value; + +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value; + +SELECT DEST1.* FROM DEST1; +SELECT DEST2.* FROM DEST2; +-- union18.q + +CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE; + +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value; + +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value; + +SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value; +SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2; +-- union19.q + + + + +CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE; + +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value; + +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value; + +SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value; +SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2; + + + +-- union2.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; +-- union20.q + +-- SORT_BEFORE_DIFF +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); + +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); +-- union21.q + +-- SORT_BEFORE_DIFF +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; + +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; +-- union22.q + +-- SORT_QUERY_RESULTS + +create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string); +create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string); + +insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src; + +insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src; + +set hive.merge.mapfiles=false; + +set hive.auto.convert.join=true; +set hive.auto.convert.join.noconditionaltask=true; +set hive.auto.convert.join.noconditionaltask.size=10000; + +-- Since the inputs are small, it should be automatically converted to mapjoin + +explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq; + +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq; + +select * from dst_union22 where ds = '2'; +-- union23.q + +explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2; + +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2; + +-- union24.q + +-- SORT_QUERY_RESULTS + +create table src2 as select key, count(1) as count from src group by key; +create table src3 as select * from src2; +create table src4 as select * from src2; +create table src5 as select * from src2; + + +set hive.merge.mapfiles=false; +set hive.merge.mapredfiles=false; + + +explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +; + +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +; + +explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +; + +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +; + +explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +; + +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +; +-- union25.q + +create table tmp_srcpart like srcpart; + +insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11'; + +explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +; +-- union26.q + +-- SORT_QUERY_RESULTS + +EXPLAIN +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +; + +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +; + + +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +; + +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +; +-- union27.q + +-- SORT_BEFORE_DIFF +create table jackson_sev_same as select * from src; +create table dim_pho as select * from src; +create table jackson_sev_add as select * from src; +explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97; +select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97; +-- union28.q + +create table union_subq_union(key int, value string); + +explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +; + +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +; + +select * from union_subq_union order by key, value limit 20; +-- union29.q + +create table union_subq_union29(key int, value string); + +explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +; + +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +; + +select * from union_subq_union29 order by key, value limit 20; +-- union3.q + +-- SORT_BEFORE_DIFF + +explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a; + + + +CREATE TABLE union_out (id int); + +insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a; + +select * from union_out; +-- union30.q + +create table union_subq_union30(key int, value string); + +explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +; + +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +; + +select * from union_subq_union30 order by key, value limit 20; +-- union31.q + +-- SORT_QUERY_RESULTS + +drop table t1; +drop table t2; + + +create table t1 as select * from src where key < 10; +create table t2 as select * from src where key < 10; + +create table t3(key string, cnt int); +create table t4(value string, cnt int); + +explain +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value; + +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value; + +select * from t3; +select * from t4; + +create table t5(c1 string, cnt int); +create table t6(c1 string, cnt int); + +explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1; + +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1; + +select * from t5; +select * from t6; + +drop table t1; +drop table t2; + +create table t1 as select * from src where key < 10; +create table t2 as select key, count(1) as cnt from src where key < 10 group by key; + +create table t7(c1 string, cnt int); +create table t8(c1 string, cnt int); + +explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1; + +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1; + +select * from t7; +select * from t8; +-- union32.q + +-- SORT_QUERY_RESULTS + +-- This tests various union queries which have columns on one side of the query +-- being of double type and those on the other side another + +drop table if exists t1; + +drop table if exists t2; + +CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10; +CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10; + +-- Test simple union with double +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +ORDER BY key; + +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +; + +-- Test union with join on the left +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +; + +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +; + +-- Test union with join on the right +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +; + +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +; + +-- Test union with join on the left selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +; + +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +; + +-- Test union with join on the right selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +; + +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +; +-- union33.q + +-- SORT_BEFORE_DIFF +-- This tests that a UNION DISTINCT with a map only subquery on one side and a +-- subquery involving two map reduce jobs on the other runs correctly. + +drop table if exists test_src; + +CREATE TABLE test_src (key STRING, value STRING); + +EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a; + +INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a; + +SELECT COUNT(*) FROM test_src; + +EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a; + +INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a; + +SELECT COUNT(*) FROM test_src; + -- union34.q + +create table src10_1 (key string, value string); +create table src10_2 (key string, value string); +create table src10_3 (key string, value string); +create table src10_4 (key string, value string); + +from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select *; + +set hive.auto.convert.join=true; +-- When we convert the Join of sub1 and sub0 into a MapJoin, +-- we can use a single MR job to evaluate this entire query. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key; + +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key; + +set hive.auto.convert.join=false; +-- When we do not convert the Join of sub1 and sub0 into a MapJoin, +-- we need to use two MR jobs to evaluate this query. +-- The first job is for the Join of sub1 and sub2. The second job +-- is for the UNION DISTINCT and ORDER BY. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key; + +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key; +-- union4.q + + + +-- union case: both subqueries are map-reduce jobs on same input, followed by filesink + +drop table if exists tmptable; + +create table tmptable(key string, value int); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc; + +insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc; + +select * from tmptable x sort by x.key; + + +-- union5.q + + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; + +select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; +-- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable; + +create table tmptable(key string, value string); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +select * from tmptable x sort by x.key, x.value; + + +-- union7.q + + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; + +select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; + + +-- union8.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + +select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; +-- union9.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; diff --git a/ql/src/test/queries/clientpositive/unionDistinct_2.q b/ql/src/test/queries/clientpositive/unionDistinct_2.q new file mode 100644 index 0000000..1c8b331 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionDistinct_2.q @@ -0,0 +1,99 @@ +CREATE TABLE u1 as select key, value from src order by key limit 5; + +CREATE TABLE u2 as select key, value from src order by key limit 3; + +CREATE TABLE u3 as select key, value from src order by key desc limit 5; + +select * from u1; + +select * from u2; + +select * from u3; + +select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab; + +select key, value from +( +select key, value from u1 +union +select key, value from u2 +union all +select key, value from u3 +) tab; + +select key, value from +( +select key, value from u1 +union distinct +select key, value from u2 +union all +select key as key, value from u3 +) tab; + +select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union +select key, value from u3 +) tab; + +select key, value from +( +select key, value from u1 +union +select key, value from u2 +union +select key as key, value from u3 +) tab; + +select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab; + +select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab; + +drop view if exists v; + +create view v as select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab; + +describe extended v; + +select * from v; + +select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union +select key as key, value from u3 +) tab; + diff --git a/ql/src/test/queries/clientpositive/union_null.q b/ql/src/test/queries/clientpositive/union_null.q index 64e1672..a17325c 100644 --- a/ql/src/test/queries/clientpositive/union_null.q +++ b/ql/src/test/queries/clientpositive/union_null.q @@ -1,7 +1,7 @@ -- SORT_BEFORE_DIFF -- HIVE-2901 -select x from (select value as x from src order by x limit 5 union all select NULL as x from src limit 5) a; +select x from (select * from (select value as x from src order by x limit 5)a union all select * from (select NULL as x from src limit 5)b )a; -- HIVE-4837 -select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a; +select * from (select * from (select null as N from src1 group by key)a UNION ALL select * from (select null as N from src1 group by key)b ) a; diff --git a/ql/src/test/queries/clientpositive/union_remove_25.q b/ql/src/test/queries/clientpositive/union_remove_25.q index c6c09e1..27d9ebe 100644 --- a/ql/src/test/queries/clientpositive/union_remove_25.q +++ b/ql/src/test/queries/clientpositive/union_remove_25.q @@ -51,17 +51,17 @@ explain insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a; insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a; show partitions outputTbl2; @@ -70,17 +70,17 @@ desc formatted outputTbl2 partition(ds='2008-04-08'); explain insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a; insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a; show partitions outputTbl3; diff --git a/ql/src/test/queries/clientpositive/union_top_level.q b/ql/src/test/queries/clientpositive/union_top_level.q index 946473a..1101a45 100644 --- a/ql/src/test/queries/clientpositive/union_top_level.q +++ b/ql/src/test/queries/clientpositive/union_top_level.q @@ -2,42 +2,42 @@ -- top level explain -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; explain -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10; +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b; -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10; +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b; -- ctas explain create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; select * from union_top; @@ -46,53 +46,53 @@ truncate table union_top; -- insert into explain insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; select * from union_top; explain insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; select * from union_top; -- create view explain create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3; +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c; select * from union_top_view; diff --git a/ql/src/test/queries/clientpositive/vector_multi_insert.q b/ql/src/test/queries/clientpositive/vector_multi_insert.q index 77404e9..9620f92 100644 --- a/ql/src/test/queries/clientpositive/vector_multi_insert.q +++ b/ql/src/test/queries/clientpositive/vector_multi_insert.q @@ -8,11 +8,11 @@ create table orc1 select rn from ( - select cast(1 as int) as rn from src limit 1 + select * from (select cast(1 as int) as rn from src limit 1)a union all - select cast(100 as int) as rn from src limit 1 + select * from (select cast(100 as int) as rn from src limit 1)b union all - select cast(10000 as int) as rn from src limit 1 + select * from (select cast(10000 as int) as rn from src limit 1)c ) t; create table orc_rn1 (rn int); diff --git a/ql/src/test/results/clientnegative/unionClusterBy.q.out b/ql/src/test/results/clientnegative/unionClusterBy.q.out new file mode 100644 index 0000000..4705f3b --- /dev/null +++ b/ql/src/test/results/clientnegative/unionClusterBy.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 6:19 Failed to recognize predicate ''. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement diff --git a/ql/src/test/results/clientnegative/unionDistributeBy.q.out b/ql/src/test/results/clientnegative/unionDistributeBy.q.out new file mode 100644 index 0000000..4705f3b --- /dev/null +++ b/ql/src/test/results/clientnegative/unionDistributeBy.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 6:19 Failed to recognize predicate ''. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement diff --git a/ql/src/test/results/clientnegative/unionLimit.q.out b/ql/src/test/results/clientnegative/unionLimit.q.out new file mode 100644 index 0000000..4705f3b --- /dev/null +++ b/ql/src/test/results/clientnegative/unionLimit.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 6:19 Failed to recognize predicate ''. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement diff --git a/ql/src/test/results/clientnegative/unionOrderBy.q.out b/ql/src/test/results/clientnegative/unionOrderBy.q.out new file mode 100644 index 0000000..4705f3b --- /dev/null +++ b/ql/src/test/results/clientnegative/unionOrderBy.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 6:19 Failed to recognize predicate ''. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement diff --git a/ql/src/test/results/clientnegative/unionSortBy.q.out b/ql/src/test/results/clientnegative/unionSortBy.q.out new file mode 100644 index 0000000..4705f3b --- /dev/null +++ b/ql/src/test/results/clientnegative/unionSortBy.q.out @@ -0,0 +1 @@ +FAILED: ParseException line 6:19 Failed to recognize predicate ''. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement diff --git a/ql/src/test/results/clientpositive/ba_table_union.q.out b/ql/src/test/results/clientpositive/ba_table_union.q.out index 706a537..639ffda 100644 --- a/ql/src/test/results/clientpositive/ba_table_union.q.out +++ b/ql/src/test/results/clientpositive/ba_table_union.q.out @@ -49,9 +49,9 @@ POSTHOOK: Input: default@src 11 12 12 -128 -145 -146 +15 +15 +153 PREHOOK: query: drop table ba_test PREHOOK: type: DROPTABLE PREHOOK: Input: default@ba_test diff --git a/ql/src/test/results/clientpositive/cbo_union.q.out b/ql/src/test/results/clientpositive/cbo_union.q.out index 1fd88ec..fb86d22 100644 --- a/ql/src/test/results/clientpositive/cbo_union.q.out +++ b/ql/src/test/results/clientpositive/cbo_union.q.out @@ -1,7 +1,7 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- 11. Union All -select * from cbo_t1 order by key, c_boolean, value, dt union all select * from cbo_t2 order by key, c_boolean, value, dt +select * from (select * from cbo_t1 order by key, c_boolean, value, dt)a union all select * from (select * from cbo_t2 order by key, c_boolean, value, dt)b PREHOOK: type: QUERY PREHOOK: Input: default@cbo_t1 PREHOOK: Input: default@cbo_t1@dt=2014 @@ -11,7 +11,7 @@ PREHOOK: Input: default@cbo_t2@dt=2014 POSTHOOK: query: -- SORT_QUERY_RESULTS -- 11. Union All -select * from cbo_t1 order by key, c_boolean, value, dt union all select * from cbo_t2 order by key, c_boolean, value, dt +select * from (select * from cbo_t1 order by key, c_boolean, value, dt)a union all select * from (select * from cbo_t2 order by key, c_boolean, value, dt)b POSTHOOK: type: QUERY POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 diff --git a/ql/src/test/results/clientpositive/char_union1.q.out b/ql/src/test/results/clientpositive/char_union1.q.out index bdc4a1d..16fffb6 100644 --- a/ql/src/test/results/clientpositive/char_union1.q.out +++ b/ql/src/test/results/clientpositive/char_union1.q.out @@ -95,9 +95,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@char_union1_ch1 #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: -- union char with different length char select * from ( select * from char_union1_ch1 @@ -119,9 +116,6 @@ POSTHOOK: Input: default@char_union1_ch1 POSTHOOK: Input: default@char_union1_ch2 #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: -- union char with string select * from ( select * from char_union1_ch1 @@ -143,9 +137,6 @@ POSTHOOK: Input: default@char_union1_ch1 POSTHOOK: Input: default@char_union1_str #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: drop table char_union1_ch1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@char_union1_ch1 diff --git a/ql/src/test/results/clientpositive/explain_logical.q.out b/ql/src/test/results/clientpositive/explain_logical.q.out index 2e73a89..8c27ef1 100644 --- a/ql/src/test/results/clientpositive/explain_logical.q.out +++ b/ql/src/test/results/clientpositive/explain_logical.q.out @@ -237,7 +237,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/explode_null.q.out b/ql/src/test/results/clientpositive/explode_null.q.out index db71c69..f0b1b3c 100644 --- a/ql/src/test/results/clientpositive/explode_null.q.out +++ b/ql/src/test/results/clientpositive/explode_null.q.out @@ -1,14 +1,14 @@ PREHOOK: query: SELECT explode(col) AS myCol FROM - (SELECT array(1,2,3) AS col FROM src LIMIT 1 + (select * from (SELECT array(1,2,3) AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1) a + select * from (SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1)eb) a PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: SELECT explode(col) AS myCol FROM - (SELECT array(1,2,3) AS col FROM src LIMIT 1 + (select * from (SELECT array(1,2,3) AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1) a + select * from (SELECT IF(false, array(1,2,3), NULL) AS col FROM src LIMIT 1)eb) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -16,16 +16,16 @@ POSTHOOK: Input: default@src 2 3 PREHOOK: query: SELECT explode(col) AS (myCol1,myCol2) FROM - (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1 + (select * from (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1) a + select * from (SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1)eb ) a PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: SELECT explode(col) AS (myCol1,myCol2) FROM - (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1 + (select * from (SELECT map(1,'one',2,'two',3,'three') AS col FROM src LIMIT 1)ea UNION ALL - SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1) a + select * from (SELECT IF(false, map(1,'one',2,'two',3,'three'), NULL) AS col FROM src LIMIT 1)eb ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index dd450cb..ca4e9aa 100644 --- a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -2406,7 +2406,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -2903,7 +2903,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index 2f08999..d6edae9 100644 --- a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -2731,7 +2731,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -3228,7 +3228,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/input25.q.out b/ql/src/test/results/clientpositive/input25.q.out index 141a576..5ba72f3 100644 --- a/ql/src/test/results/clientpositive/input25.q.out +++ b/ql/src/test/results/clientpositive/input25.q.out @@ -22,16 +22,16 @@ POSTHOOK: Output: default@tst POSTHOOK: Output: default@tst@d=2009-02-02 PREHOOK: query: explain select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq PREHOOK: type: QUERY POSTHOOK: query: explain select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -152,9 +152,9 @@ STAGE PLANS: ListSink PREHOOK: query: select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq PREHOOK: type: QUERY PREHOOK: Input: default@tst @@ -162,9 +162,9 @@ PREHOOK: Input: default@tst@d=2009-01-01 PREHOOK: Input: default@tst@d=2009-02-02 #### A masked pattern was here #### POSTHOOK: query: select * from ( - select * from tst x where x.d='2009-01-01' limit 10 + select * from (select * from tst x where x.d='2009-01-01' limit 10)a union all - select * from tst x where x.d='2009-02-02' limit 10 + select * from (select * from tst x where x.d='2009-02-02' limit 10)b ) subq POSTHOOK: type: QUERY POSTHOOK: Input: default@tst diff --git a/ql/src/test/results/clientpositive/input26.q.out b/ql/src/test/results/clientpositive/input26.q.out index 66d3bd2..ef38d49 100644 --- a/ql/src/test/results/clientpositive/input26.q.out +++ b/ql/src/test/results/clientpositive/input26.q.out @@ -1,15 +1,15 @@ PREHOOK: query: explain select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq PREHOOK: type: QUERY POSTHOOK: query: explain select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -113,18 +113,18 @@ STAGE PLANS: ListSink PREHOOK: query: select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 #### A masked pattern was here #### POSTHOOK: query: select * from ( - select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5 + select * from (select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit 5)pa union all - select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5 + select * from (select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5)pb )subq POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/input_part7.q.out b/ql/src/test/results/clientpositive/input_part7.q.out index 6094f9c..fea1759 100644 --- a/ql/src/test/results/clientpositive/input_part7.q.out +++ b/ql/src/test/results/clientpositive/input_part7.q.out @@ -19,7 +19,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/join34.q.out b/ql/src/test/results/clientpositive/join34.q.out index a20e49f..48c3b74 100644 --- a/ql/src/test/results/clientpositive/join34.q.out +++ b/ql/src/test/results/clientpositive/join34.q.out @@ -40,7 +40,7 @@ TOK_QUERY TOK_FROM TOK_JOIN TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/join35.q.out b/ql/src/test/results/clientpositive/join35.q.out index 937539c..2be3939 100644 --- a/ql/src/test/results/clientpositive/join35.q.out +++ b/ql/src/test/results/clientpositive/join35.q.out @@ -40,7 +40,7 @@ TOK_QUERY TOK_FROM TOK_JOIN TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/load_dyn_part14.q.out b/ql/src/test/results/clientpositive/load_dyn_part14.q.out index a9dde4d..e5fe51f 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part14.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part14.q.out @@ -36,21 +36,21 @@ value string PREHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -256,22 +256,22 @@ STAGE PLANS: PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@nzhang_part14 POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY POSTHOOK: Input: default@src diff --git a/ql/src/test/results/clientpositive/merge4.q.out b/ql/src/test/results/clientpositive/merge4.q.out index 121b724..031376f 100644 --- a/ql/src/test/results/clientpositive/merge4.q.out +++ b/ql/src/test/results/clientpositive/merge4.q.out @@ -2762,34 +2762,51 @@ select * from ( select '1' as key, '1' as value, 'file,' as hr from src limit 1) s POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-9 is a root stage - Stage-2 depends on stages: Stage-9 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-9 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1500 Data size: 140124 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '1' (type: string), '1' (type: string), 'file,' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 129500 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Union + Statistics: Num rows: 1500 Data size: 140124 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) @@ -2800,47 +2817,17 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Statistics: Num rows: 1 Data size: 259 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1001 Data size: 10883 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1001 Data size: 10883 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part - TableScan - Union - Statistics: Num rows: 1001 Data size: 10883 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1001 Data size: 10883 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part - Stage: Stage-8 + Stage: Stage-7 Conditional Operator - Stage: Stage-5 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -2859,10 +2846,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-3 + Stage: Stage-2 Stats-Aggr Operator - Stage: Stage-4 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -2874,7 +2861,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-6 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -2886,7 +2873,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part - Stage: Stage-7 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -2913,13 +2900,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@nzhang_part@ds=2010-08-15/hr=11 -POSTHOOK: Output: default@nzhang_part@ds=2010-08-15/hr=12 POSTHOOK: Output: default@nzhang_part@ds=2010-08-15/hr=file, -POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=11).value EXPRESSION [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=12).value EXPRESSION [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=file,).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part PARTITION(ds=2010-08-15,hr=file,).value EXPRESSION [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: show partitions nzhang_part diff --git a/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out b/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out index 1fcbc0a..1671c6b 100644 --- a/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out +++ b/ql/src/test/results/clientpositive/metadataOnlyOptimizer.q.out @@ -1332,24 +1332,23 @@ POSTHOOK: Input: default@src 96 97 98 -PREHOOK: query: select k from (SELECT '1' as k from src limit 0 union all select key as k from src limit 1)tab +PREHOOK: query: select k from (select * from (SELECT '1' as k from src limit 0)a union all select key as k from src limit 1)tab PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select k from (SELECT '1' as k from src limit 0 union all select key as k from src limit 1)tab +POSTHOOK: query: select k from (select * from (SELECT '1' as k from src limit 0)a union all select key as k from src limit 1)tab POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 238 -PREHOOK: query: select k from (SELECT '1' as k from src limit 1 union all select key as k from src limit 0)tab +PREHOOK: query: select k from (select * from (SELECT '1' as k from src limit 1)a union all select key as k from src limit 0)tab PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select k from (SELECT '1' as k from src limit 1 union all select key as k from src limit 0)tab +POSTHOOK: query: select k from (select * from (SELECT '1' as k from src limit 1)a union all select key as k from src limit 0)tab POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -1 PREHOOK: query: select max(ds) from srcpart PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/optimize_nullscan.q.out index 4eb498e..8b4ebdd 100644 --- a/ql/src/test/results/clientpositive/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/optimize_nullscan.q.out @@ -703,7 +703,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -1690,7 +1690,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/script_env_var1.q.out b/ql/src/test/results/clientpositive/script_env_var1.q.out index 8e1075a..5ab1c20 100644 --- a/ql/src/test/results/clientpositive/script_env_var1.q.out +++ b/ql/src/test/results/clientpositive/script_env_var1.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/script_env_var2.q.out b/ql/src/test/results/clientpositive/script_env_var2.q.out index 89f3606..c3bb990 100644 --- a/ql/src/test/results/clientpositive/script_env_var2.q.out +++ b/ql/src/test/results/clientpositive/script_env_var2.q.out @@ -1,14 +1,14 @@ PREHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 569501f..271524a 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -1888,7 +1888,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -2241,7 +2241,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 6e66697..5cbae1f 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -1960,7 +1960,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -2313,7 +2313,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/join34.q.out b/ql/src/test/results/clientpositive/spark/join34.q.out index c337093..055f198 100644 --- a/ql/src/test/results/clientpositive/spark/join34.q.out +++ b/ql/src/test/results/clientpositive/spark/join34.q.out @@ -40,7 +40,7 @@ TOK_QUERY TOK_FROM TOK_JOIN TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/join35.q.out b/ql/src/test/results/clientpositive/spark/join35.q.out index 2b217c1..39dcf5b 100644 --- a/ql/src/test/results/clientpositive/spark/join35.q.out +++ b/ql/src/test/results/clientpositive/spark/join35.q.out @@ -40,7 +40,7 @@ TOK_QUERY TOK_FROM TOK_JOIN TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out index 1f9985f..66db7bd 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out @@ -36,21 +36,21 @@ value string PREHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -175,22 +175,22 @@ STAGE PLANS: PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@nzhang_part14 POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY POSTHOOK: Input: default@src diff --git a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out index 3a8efcf..216726a 100644 --- a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out @@ -599,7 +599,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -1457,7 +1457,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/script_env_var1.q.out b/ql/src/test/results/clientpositive/spark/script_env_var1.q.out index 8e1075a..5ab1c20 100644 --- a/ql/src/test/results/clientpositive/spark/script_env_var1.q.out +++ b/ql/src/test/results/clientpositive/spark/script_env_var1.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/script_env_var2.q.out b/ql/src/test/results/clientpositive/spark/script_env_var2.q.out index 89f3606..c3bb990 100644 --- a/ql/src/test/results/clientpositive/spark/script_env_var2.q.out +++ b/ql/src/test/results/clientpositive/spark/script_env_var2.q.out @@ -1,14 +1,14 @@ PREHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union3.q.out b/ql/src/test/results/clientpositive/spark/union3.q.out index 1e79c34..21b6da5 100644 --- a/ql/src/test/results/clientpositive/spark/union3.q.out +++ b/ql/src/test/results/clientpositive/spark/union3.q.out @@ -5,17 +5,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY POSTHOOK: query: -- SORT_BEFORE_DIFF @@ -25,17 +24,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -46,12 +44,11 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 10 <- Map 9 (GROUP, 1) Reducer 2 <- Map 1 (GROUP, 1) Reducer 5 <- Map 4 (GROUP, 1) - Reducer 8 <- Map 7 (GROUP, 1) - Reducer 3 <- Reducer 2 (SORT, 1) - Reducer 6 <- Reducer 5 (SORT, 1) + Reducer 7 <- Map 6 (GROUP, 1) + Reducer 9 <- Map 8 (GROUP, 1) + Reducer 3 <- Reducer 2 (SORT, 1), Reducer 5 (SORT, 1), Reducer 7 (SORT, 1), Reducer 9 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -80,7 +77,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 7 + Map 6 Map Operator Tree: TableScan alias: src @@ -93,7 +90,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 9 + Map 8 Map Operator Tree: TableScan alias: src @@ -106,86 +103,63 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 10 + Reducer 2 Reduce Operator Tree: Limit Number of rows: 1 Select Operator - expressions: 4 (type: int) + expressions: 1 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 2 - Reduce Operator Tree: - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: 1 (type: int) - sort order: + - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 5 Reduce Operator Tree: Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: 2 (type: int) - sort order: + - Map-reduce partition columns: 2 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 6 - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 Select Operator - expressions: _col0 (type: int) + expressions: 2 (type: int) outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 7 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 3 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 9 + Reduce Operator Tree: + Limit + Number of rows: 1 + Select Operator + expressions: 4 (type: int) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Stage: Stage-0 Fetch Operator @@ -206,17 +180,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -226,27 +199,26 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_out POSTHOOK: Lineage: union_out.id EXPRESSION [] -PREHOOK: query: select * from union_out cluster by id +PREHOOK: query: select * from union_out PREHOOK: type: QUERY PREHOOK: Input: default@union_out #### A masked pattern was here #### -POSTHOOK: query: select * from union_out cluster by id +POSTHOOK: query: select * from union_out POSTHOOK: type: QUERY POSTHOOK: Input: default@union_out #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_null.q.out b/ql/src/test/results/clientpositive/spark/union_null.q.out index 4574a2e..32cdf65 100644 --- a/ql/src/test/results/clientpositive/spark/union_null.q.out +++ b/ql/src/test/results/clientpositive/spark/union_null.q.out @@ -1,34 +1,34 @@ PREHOOK: query: -- SORT_BEFORE_DIFF -- HIVE-2901 -select x from (select value as x from src order by x limit 5 union all select NULL as x from src limit 5) a +select x from (select * from (select value as x from src order by x limit 5)a union all select * from (select NULL as x from src limit 5)b )a PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- SORT_BEFORE_DIFF -- HIVE-2901 -select x from (select value as x from src order by x limit 5 union all select NULL as x from src limit 5) a +select x from (select * from (select value as x from src order by x limit 5)a union all select * from (select NULL as x from src limit 5)b )a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL val_0 val_0 val_0 val_10 val_100 +NULL +NULL +NULL +NULL +NULL PREHOOK: query: -- HIVE-4837 -select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a +select * from (select * from (select null as N from src1 group by key)a UNION ALL select * from (select null as N from src1 group by key)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### POSTHOOK: query: -- HIVE-4837 -select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a +select * from (select * from (select null as N from src1 group by key)a UNION ALL select * from (select null as N from src1 group by key)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_ppr.q.out b/ql/src/test/results/clientpositive/spark/union_ppr.q.out index 6204de4..0a1e6c4 100644 --- a/ql/src/test/results/clientpositive/spark/union_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/union_ppr.q.out @@ -21,7 +21,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out index d36a246..286a3ce 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out @@ -258,18 +258,18 @@ PREHOOK: query: explain insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -366,9 +366,9 @@ STAGE PLANS: PREHOOK: query: insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -378,9 +378,9 @@ PREHOOK: Output: default@outputtbl2 POSTHOOK: query: insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart @@ -440,17 +440,17 @@ Storage Desc Params: PREHOOK: query: explain insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -548,9 +548,9 @@ STAGE PLANS: PREHOOK: query: insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -560,9 +560,9 @@ PREHOOK: Output: default@outputtbl3 POSTHOOK: query: insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/tez/cbo_union.q.out b/ql/src/test/results/clientpositive/tez/cbo_union.q.out index 1fd88ec..fb86d22 100644 --- a/ql/src/test/results/clientpositive/tez/cbo_union.q.out +++ b/ql/src/test/results/clientpositive/tez/cbo_union.q.out @@ -1,7 +1,7 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- 11. Union All -select * from cbo_t1 order by key, c_boolean, value, dt union all select * from cbo_t2 order by key, c_boolean, value, dt +select * from (select * from cbo_t1 order by key, c_boolean, value, dt)a union all select * from (select * from cbo_t2 order by key, c_boolean, value, dt)b PREHOOK: type: QUERY PREHOOK: Input: default@cbo_t1 PREHOOK: Input: default@cbo_t1@dt=2014 @@ -11,7 +11,7 @@ PREHOOK: Input: default@cbo_t2@dt=2014 POSTHOOK: query: -- SORT_QUERY_RESULTS -- 11. Union All -select * from cbo_t1 order by key, c_boolean, value, dt union all select * from cbo_t2 order by key, c_boolean, value, dt +select * from (select * from cbo_t1 order by key, c_boolean, value, dt)a union all select * from (select * from cbo_t2 order by key, c_boolean, value, dt)b POSTHOOK: type: QUERY POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 diff --git a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out index da456c7..0131534 100644 --- a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out @@ -597,7 +597,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -1458,7 +1458,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/tez/script_env_var1.q.out b/ql/src/test/results/clientpositive/tez/script_env_var1.q.out index 8e1075a..5ab1c20 100644 --- a/ql/src/test/results/clientpositive/tez/script_env_var1.q.out +++ b/ql/src/test/results/clientpositive/tez/script_env_var1.q.out @@ -1,16 +1,16 @@ PREHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Verifies that script operator ID environment variables have unique values -- in each instance of the script operator. SELECT count(1) FROM -( SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)x UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src LIMIT 1)y ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/script_env_var2.q.out b/ql/src/test/results/clientpositive/tez/script_env_var2.q.out index 89f3606..c3bb990 100644 --- a/ql/src/test/results/clientpositive/tez/script_env_var2.q.out +++ b/ql/src/test/results/clientpositive/tez/script_env_var2.q.out @@ -1,14 +1,14 @@ PREHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- Same test as script_env_var1, but test setting the variable name SELECT count(1) FROM -( SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 UNION ALL - SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1 ) a GROUP BY key +( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL + SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/union3.q.out b/ql/src/test/results/clientpositive/tez/union3.q.out index 3dad621..1de399c 100644 --- a/ql/src/test/results/clientpositive/tez/union3.q.out +++ b/ql/src/test/results/clientpositive/tez/union3.q.out @@ -5,17 +5,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY POSTHOOK: query: -- SORT_BEFORE_DIFF @@ -25,17 +24,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -46,12 +44,11 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 @@ -67,7 +64,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 10 + Map 5 Map Operator Tree: TableScan alias: src @@ -80,7 +77,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 5 + Map 7 Map Operator Tree: TableScan alias: src @@ -93,7 +90,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 8 + Map 9 Map Operator Tree: TableScan alias: src @@ -106,88 +103,65 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 11 + Reducer 10 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 4 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Reducer 2 Reduce Operator Tree: Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: 1 (type: int) - sort order: + - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 3 + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 4 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 6 Reduce Operator Tree: Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: 2 (type: int) - sort order: + - Map-reduce partition columns: 2 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 7 - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 Select Operator - expressions: _col0 (type: int) + expressions: 2 (type: int) outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 8 Reduce Operator Tree: Limit Number of rows: 1 Select Operator expressions: 3 (type: int) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Union 3 + Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -208,17 +182,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -228,27 +201,26 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_out POSTHOOK: Lineage: union_out.id EXPRESSION [] -PREHOOK: query: select * from union_out cluster by id +PREHOOK: query: select * from union_out PREHOOK: type: QUERY PREHOOK: Input: default@union_out #### A masked pattern was here #### -POSTHOOK: query: select * from union_out cluster by id +POSTHOOK: query: select * from union_out POSTHOOK: type: QUERY POSTHOOK: Input: default@union_out #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out new file mode 100644 index 0000000..6e53ecd --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/unionDistinct_1.q.out @@ -0,0 +1,15535 @@ +PREHOOK: query: -- union10.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + +create table tmptable(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union10.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + +create table tmptable(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 9 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 10 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [] +POSTHOOK: Lineage: tmptable.value EXPRESSION [] +PREHOOK: query: select * from tmptable x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### +tst1 500 +tst2 500 +tst3 500 +PREHOOK: query: -- union11.q + + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union11.q + + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 9 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 10 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +tst3 1 +PREHOOK: query: -- union12.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink + +create table tmptable12(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable12 +POSTHOOK: query: -- union12.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink + +create table tmptable12(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable12 +PREHOOK: query: explain +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 9 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 10 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable12 + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable12 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@tmptable12 +POSTHOOK: query: insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@tmptable12 +POSTHOOK: Lineage: tmptable12.key EXPRESSION [] +POSTHOOK: Lineage: tmptable12.value EXPRESSION [] +PREHOOK: query: select * from tmptable12 x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable12 +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable12 x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable12 +#### A masked pattern was here #### +tst1 500 +tst2 25 +tst3 1000 +PREHOOK: query: -- union13.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are a map-only jobs, same input, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union13.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are a map-only jobs, same input, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union14.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union14.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union15.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union15.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 7 <- Union 3 (CONTAINS) + Map 8 <- Union 5 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 8 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union16.q + +-- SORT_BEFORE_DIFF +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +POSTHOOK: query: -- union16.q + +-- SORT_BEFORE_DIFF +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 51 <- Union 2 (CONTAINS) + Map 52 <- Union 4 (CONTAINS) + Map 53 <- Union 6 (CONTAINS) + Map 54 <- Union 8 (CONTAINS) + Map 55 <- Union 10 (CONTAINS) + Map 56 <- Union 12 (CONTAINS) + Map 57 <- Union 14 (CONTAINS) + Map 58 <- Union 16 (CONTAINS) + Map 59 <- Union 18 (CONTAINS) + Map 60 <- Union 20 (CONTAINS) + Map 61 <- Union 22 (CONTAINS) + Map 62 <- Union 24 (CONTAINS) + Map 63 <- Union 26 (CONTAINS) + Map 64 <- Union 28 (CONTAINS) + Map 65 <- Union 30 (CONTAINS) + Map 66 <- Union 32 (CONTAINS) + Map 67 <- Union 34 (CONTAINS) + Map 68 <- Union 36 (CONTAINS) + Map 69 <- Union 38 (CONTAINS) + Map 70 <- Union 40 (CONTAINS) + Map 71 <- Union 42 (CONTAINS) + Map 72 <- Union 44 (CONTAINS) + Map 73 <- Union 46 (CONTAINS) + Map 74 <- Union 48 (CONTAINS) + Reducer 11 <- Union 10 (SIMPLE_EDGE), Union 12 (CONTAINS) + Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) + Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) + Reducer 17 <- Union 16 (SIMPLE_EDGE), Union 18 (CONTAINS) + Reducer 19 <- Union 18 (SIMPLE_EDGE), Union 20 (CONTAINS) + Reducer 21 <- Union 20 (SIMPLE_EDGE), Union 22 (CONTAINS) + Reducer 23 <- Union 22 (SIMPLE_EDGE), Union 24 (CONTAINS) + Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) + Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) + Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 31 <- Union 30 (SIMPLE_EDGE), Union 32 (CONTAINS) + Reducer 33 <- Union 32 (SIMPLE_EDGE), Union 34 (CONTAINS) + Reducer 35 <- Union 34 (SIMPLE_EDGE), Union 36 (CONTAINS) + Reducer 37 <- Union 36 (SIMPLE_EDGE), Union 38 (CONTAINS) + Reducer 39 <- Union 38 (SIMPLE_EDGE), Union 40 (CONTAINS) + Reducer 41 <- Union 40 (SIMPLE_EDGE), Union 42 (CONTAINS) + Reducer 43 <- Union 42 (SIMPLE_EDGE), Union 44 (CONTAINS) + Reducer 45 <- Union 44 (SIMPLE_EDGE), Union 46 (CONTAINS) + Reducer 47 <- Union 46 (SIMPLE_EDGE), Union 48 (CONTAINS) + Reducer 49 <- Union 48 (SIMPLE_EDGE) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 50 <- Reducer 49 (SIMPLE_EDGE) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 10 (CONTAINS), Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 51 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 52 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 53 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 54 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 55 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 56 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 57 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 58 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 59 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 60 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 61 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 62 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 63 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 64 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 65 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 66 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 67 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 68 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 69 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 70 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 71 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 72 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 73 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 74 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 11 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 13 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 15 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 17 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 19 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 21 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 23 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 25 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 27 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 29 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 31 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 33 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 35 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 37 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 39 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 41 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 43 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 45 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 47 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 49 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 50 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 9 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 10 + Vertex: Union 10 + Union 12 + Vertex: Union 12 + Union 14 + Vertex: Union 14 + Union 16 + Vertex: Union 16 + Union 18 + Vertex: Union 18 + Union 2 + Vertex: Union 2 + Union 20 + Vertex: Union 20 + Union 22 + Vertex: Union 22 + Union 24 + Vertex: Union 24 + Union 26 + Vertex: Union 26 + Union 28 + Vertex: Union 28 + Union 30 + Vertex: Union 30 + Union 32 + Vertex: Union 32 + Union 34 + Vertex: Union 34 + Union 36 + Vertex: Union 36 + Union 38 + Vertex: Union 38 + Union 4 + Vertex: Union 4 + Union 40 + Vertex: Union 40 + Union 42 + Vertex: Union 42 + Union 44 + Vertex: Union 44 + Union 46 + Vertex: Union 46 + Union 48 + Vertex: Union 48 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: -- union17.q + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST1 +POSTHOOK: query: -- union17.q + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST1 +PREHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST2 +POSTHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST2 +PREHOOK: query: -- SORT_BEFORE_DIFF +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_BEFORE_DIFF +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Map 8 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 8 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: substr(_col1, 5) (type: string) + sort order: + + Map-reduce partition columns: substr(_col1, 5) (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Forward + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(DISTINCT KEY._col0) + keys: VALUE._col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Group By Operator + aggregations: count(DISTINCT KEY._col0) + keys: VALUE._col0 (type: string), VALUE._col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1 +PREHOOK: Output: default@dest2 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1 +POSTHOOK: Output: default@dest2 +POSTHOOK: Lineage: dest1.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST1.* FROM DEST1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST1.* FROM DEST1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST2.* FROM DEST2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST2.* FROM DEST2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest2 +#### A masked pattern was here #### +0 val_0 1 +10 val_10 1 +100 val_100 1 +103 val_103 1 +104 val_104 1 +105 val_105 1 +11 val_11 1 +111 val_111 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +119 val_119 1 +12 val_12 1 +120 val_120 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +149 val_149 1 +15 val_15 1 +150 val_150 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +17 val_17 1 +170 val_170 1 +172 val_172 1 +174 val_174 1 +175 val_175 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +18 val_18 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +189 val_189 1 +19 val_19 1 +190 val_190 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +199 val_199 1 +2 val_2 1 +20 val_20 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +205 val_205 1 +207 val_207 1 +208 val_208 1 +209 val_209 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +230 val_230 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +238 val_238 1 +239 val_239 1 +24 val_24 1 +241 val_241 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +26 val_26 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +266 val_266 1 +27 val_27 1 +272 val_272 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +278 val_278 1 +28 val_28 1 +280 val_280 1 +281 val_281 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +30 val_30 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +317 val_317 1 +318 val_318 1 +321 val_321 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +327 val_327 1 +33 val_33 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +34 val_34 1 +341 val_341 1 +342 val_342 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +35 val_35 1 +351 val_351 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +37 val_37 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +396 val_396 1 +397 val_397 1 +399 val_399 1 +4 val_4 1 +400 val_400 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +404 val_404 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +41 val_41 1 +411 val_411 1 +413 val_413 1 +414 val_414 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +42 val_42 1 +421 val_421 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +43 val_43 1 +430 val_430 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +439 val_439 1 +44 val_44 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +463 val_463 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +469 val_469 1 +47 val_47 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +5 val_5 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +8 val_8 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +9 val_9 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +98 val_98 1 +tst1 500 1 +PREHOOK: query: -- union18.q + +CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST118 +POSTHOOK: query: -- union18.q + +CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST118 +PREHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST218 +POSTHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST218 +PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest118 +PREHOOK: Output: default@dest218 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest118 +POSTHOOK: Output: default@dest218 +POSTHOOK: Lineage: dest118.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest118.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest118 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest118 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +tst1 500 +PREHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest218 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest218 +#### A masked pattern was here #### +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: -- union19.q + + + + +CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST119 +POSTHOOK: query: -- union19.q + + + + +CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST119 +PREHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST219 +POSTHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST219 +PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest119 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest119 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest119 +PREHOOK: Output: default@dest219 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest119 +POSTHOOK: Output: default@dest219 +POSTHOOK: Lineage: dest119.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest119.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest119 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest119 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest219 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest219 +#### A masked pattern was here #### +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: -- union2.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union2.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: -- union20.q + +-- SORT_BEFORE_DIFF +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +POSTHOOK: query: -- union20.q + +-- SORT_BEFORE_DIFF +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 11 <- Union 9 (CONTAINS) + Map 6 <- Union 3 (CONTAINS) + Reducer 10 <- Union 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 9 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 11 + Map Operator Tree: + TableScan + alias: s1 + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: s1 + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 10 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 3 + Vertex: Union 3 + Union 9 + Vertex: Union 9 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +tst1 500 tst1 500 +PREHOOK: query: -- union21.q + +-- SORT_BEFORE_DIFF +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +POSTHOOK: query: -- union21.q + +-- SORT_BEFORE_DIFF +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 10 <- Union 2 (CONTAINS) + Map 11 <- Union 4 (CONTAINS) + Map 12 <- Union 6 (CONTAINS) + Map 13 <- Union 8 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: '1' (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Map 10 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: reverse(key) (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Map 11 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Map 12 + Map Operator Tree: + TableScan + alias: src_thrift + Select Operator + expressions: astring (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Map 13 + Map Operator Tree: + TableScan + alias: src_thrift + Select Operator + expressions: lstring[0] (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Reducer 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Reducer 9 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +NULL 1 +0 1 +001 1 +002 1 +004 1 +01 1 +013 1 +02 1 +021 1 +03 1 +032 1 +034 1 +051 1 +061 1 +062 1 +063 1 +064 1 +07 1 +071 1 +074 1 +08 1 +081 1 +082 1 +084 1 +09 1 +091 1 +094 1 +1 1 +10 1 +100 1 +102 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +122 1 +123 1 +124 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +14 1 +142 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +182 1 +183 1 +184 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +204 1 +205 1 +207 1 +208 1 +209 1 +21 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +234 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +243 1 +244 1 +247 1 +248 1 +249 1 +251 1 +252 1 +254 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +261 1 +262 1 +263 1 +264 1 +265 1 +266 1 +27 1 +271 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +29 1 +291 1 +292 1 +293 1 +294 1 +296 1 +298 1 +30 1 +301 1 +302 1 +304 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +312 1 +314 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +354 1 +356 1 +360 1 +361 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +372 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +38 1 +381 1 +382 1 +384 1 +386 1 +389 1 +391 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +40 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +412 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +422 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +442 1 +443 1 +444 1 +446 1 +448 1 +449 1 +45 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +46 1 +460 1 +461 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +471 1 +472 1 +473 1 +475 1 +477 1 +478 1 +479 1 +48 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +50 1 +501 1 +502 1 +503 1 +51 1 +513 1 +521 1 +523 1 +53 1 +532 1 +533 1 +534 1 +54 1 +541 1 +543 1 +551 1 +552 1 +554 1 +56 1 +561 1 +562 1 +563 1 +57 1 +571 1 +572 1 +573 1 +574 1 +58 1 +582 1 +584 1 +59 1 +591 1 +593 1 +594 1 +60 1 +603 1 +604 1 +611 1 +612 1 +613 1 +62 1 +621 1 +622 1 +631 1 +633 1 +634 1 +64 1 +641 1 +644 1 +65 1 +651 1 +652 1 +653 1 +66 1 +661 1 +662 1 +663 1 +664 1 +67 1 +671 1 +68 1 +681 1 +682 1 +683 1 +69 1 +691 1 +692 1 +693 1 +694 1 +70 1 +702 1 +703 1 +704 1 +71 1 +712 1 +713 1 +714 1 +72 1 +723 1 +724 1 +73 1 +731 1 +732 1 +734 1 +74 1 +742 1 +75 1 +751 1 +752 1 +754 1 +76 1 +761 1 +763 1 +764 1 +77 1 +771 1 +772 1 +773 1 +774 1 +78 1 +781 1 +782 1 +784 1 +79 1 +791 1 +793 1 +794 1 +8 1 +80 1 +802 1 +803 1 +81 1 +811 1 +812 1 +813 1 +814 1 +82 1 +821 1 +822 1 +83 1 +831 1 +832 1 +833 1 +834 1 +84 1 +842 1 +843 1 +844 1 +85 1 +851 1 +852 1 +854 1 +86 1 +861 1 +863 1 +864 1 +87 1 +871 1 +872 1 +873 1 +874 1 +882 1 +89 1 +892 1 +894 1 +9 1 +90 1 +902 1 +903 1 +904 1 +91 1 +911 1 +912 1 +914 1 +92 1 +921 1 +922 1 +924 1 +932 1 +933 1 +934 1 +941 1 +942 1 +944 1 +95 1 +954 1 +96 1 +961 1 +963 1 +964 1 +97 1 +971 1 +973 1 +974 1 +98 1 +981 1 +982 1 +983 1 +984 1 +991 1 +993 1 +record_0 1 +record_1 1 +record_2 1 +record_3 1 +record_4 1 +record_5 1 +record_6 1 +record_7 1 +record_8 1 +record_9 1 +PREHOOK: query: -- union22.q + +-- SORT_QUERY_RESULTS + +create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22 +POSTHOOK: query: -- union22.q + +-- SORT_QUERY_RESULTS + +create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22 +PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22_delta +POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22_delta +PREHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin + +explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +PREHOOK: type: QUERY +POSTHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin + +explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + dst_union22_delta + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + k1 + k1 + TOK_SELEXPR + TOK_TABLE_OR_COL + k2 + k2 + TOK_SELEXPR + TOK_TABLE_OR_COL + k3 + k3 + TOK_SELEXPR + TOK_TABLE_OR_COL + k4 + k4 + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + <= + TOK_TABLE_OR_COL + k0 + 50 + TOK_QUERY + TOK_FROM + TOK_LEFTOUTERJOIN + TOK_TABREF + TOK_TABNAME + dst_union22 + a + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + dst_union22_delta + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + > + TOK_TABLE_OR_COL + k0 + 50 + b + and + = + . + TOK_TABLE_OR_COL + a + k1 + . + TOK_TABLE_OR_COL + b + k1 + = + . + TOK_TABLE_OR_COL + a + ds + '1' + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + k1 + k1 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + k2 + k2 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + k3 + k3 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + k4 + k4 + TOK_WHERE + > + . + TOK_TABLE_OR_COL + a + k1 + 20 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + subq + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dst_union22 + TOK_PARTSPEC + TOK_PARTVAL + ds + '2' + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dst_union22_delta + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (k0 <= 50) (type: boolean) + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Map 4 + Map Operator Tree: + TableScan + alias: dst_union22_delta + Statistics: Num rows: 500 Data size: 16936 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((k0 > 50) and (k1 > 20)) (type: boolean) + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col1, _col3, _col4 + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col3 (type: string), _col4 (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [dst_union22_delta] + Map 5 + Map Operator Tree: + TableScan + alias: a + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (k1 > 20) (type: boolean) + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter mappings: + 0 [1, 1] + filter predicates: + 0 {(ds = '1')} + 1 + Estimated key counts: Map 4 => 55 + keys: + 0 k1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col11, _col12 + input vertices: + 1 Map 4 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col11 (type: string), _col12 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 11124 + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11624 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + name: default.dst_union22 + Truncated Path -> Alias: + /dst_union22/ds=1 [a] + Reducer 3 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=1 +PREHOOK: Input: default@dst_union22_delta +PREHOOK: Input: default@dst_union22_delta@ds=1 +PREHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=1 +POSTHOOK: Input: default@dst_union22_delta +POSTHOOK: Input: default@dst_union22_delta@ds=1 +POSTHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ] +PREHOOK: query: select * from dst_union22 where ds = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: select * from dst_union22 where ds = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +0 val_0 0 val_0 2 +10 val_10 10 val_10 2 +100 val_100 100 val_100 2 +103 val_103 103 val_103 2 +104 val_104 104 val_104 2 +105 val_105 105 val_105 2 +11 val_11 11 val_11 2 +111 val_111 111 val_111 2 +113 val_113 113 val_113 2 +114 val_114 114 val_114 2 +116 val_116 116 val_116 2 +118 val_118 118 val_118 2 +119 val_119 119 val_119 2 +12 val_12 12 val_12 2 +120 val_120 120 val_120 2 +125 val_125 125 val_125 2 +126 val_126 126 val_126 2 +128 val_128 128 val_128 2 +129 val_129 129 val_129 2 +131 val_131 131 val_131 2 +133 val_133 133 val_133 2 +134 val_134 134 val_134 2 +136 val_136 136 val_136 2 +137 val_137 137 val_137 2 +138 val_138 138 val_138 2 +143 val_143 143 val_143 2 +145 val_145 145 val_145 2 +146 val_146 146 val_146 2 +149 val_149 149 val_149 2 +15 val_15 15 val_15 2 +150 val_150 150 val_150 2 +152 val_152 152 val_152 2 +153 val_153 153 val_153 2 +155 val_155 155 val_155 2 +156 val_156 156 val_156 2 +157 val_157 157 val_157 2 +158 val_158 158 val_158 2 +160 val_160 160 val_160 2 +162 val_162 162 val_162 2 +163 val_163 163 val_163 2 +164 val_164 164 val_164 2 +165 val_165 165 val_165 2 +166 val_166 166 val_166 2 +167 val_167 167 val_167 2 +168 val_168 168 val_168 2 +169 val_169 169 val_169 2 +17 val_17 17 val_17 2 +170 val_170 170 val_170 2 +172 val_172 172 val_172 2 +174 val_174 174 val_174 2 +175 val_175 175 val_175 2 +176 val_176 176 val_176 2 +177 val_177 177 val_177 2 +178 val_178 178 val_178 2 +179 val_179 179 val_179 2 +18 val_18 18 val_18 2 +180 val_180 180 val_180 2 +181 val_181 181 val_181 2 +183 val_183 183 val_183 2 +186 val_186 186 val_186 2 +187 val_187 187 val_187 2 +189 val_189 189 val_189 2 +19 val_19 19 val_19 2 +190 val_190 190 val_190 2 +191 val_191 191 val_191 2 +192 val_192 192 val_192 2 +193 val_193 193 val_193 2 +194 val_194 194 val_194 2 +195 val_195 195 val_195 2 +196 val_196 196 val_196 2 +197 val_197 197 val_197 2 +199 val_199 199 val_199 2 +2 val_2 2 val_2 2 +20 val_20 20 val_20 2 +200 val_200 200 val_200 2 +201 val_201 201 val_201 2 +202 val_202 202 val_202 2 +203 val_203 203 val_203 2 +205 val_205 205 val_205 2 +207 val_207 207 val_207 2 +208 val_208 208 val_208 2 +209 val_209 209 val_209 2 +213 val_213 213 val_213 2 +214 val_214 214 val_214 2 +216 val_216 216 val_216 2 +217 val_217 217 val_217 2 +218 val_218 218 val_218 2 +219 val_219 219 val_219 2 +221 val_221 221 val_221 2 +222 val_222 222 val_222 2 +223 val_223 223 val_223 2 +224 val_224 224 val_224 2 +226 val_226 226 val_226 2 +228 val_228 228 val_228 2 +229 val_229 229 val_229 2 +230 val_230 230 val_230 2 +233 val_233 233 val_233 2 +235 val_235 235 val_235 2 +237 val_237 237 val_237 2 +238 val_238 238 val_238 2 +239 val_239 239 val_239 2 +24 val_24 NULL NULL 2 +24 val_24 24 val_24 2 +241 val_241 241 val_241 2 +242 val_242 242 val_242 2 +244 val_244 244 val_244 2 +247 val_247 247 val_247 2 +248 val_248 248 val_248 2 +249 val_249 249 val_249 2 +252 val_252 252 val_252 2 +255 val_255 255 val_255 2 +256 val_256 256 val_256 2 +257 val_257 257 val_257 2 +258 val_258 258 val_258 2 +26 val_26 NULL NULL 2 +26 val_26 26 val_26 2 +260 val_260 260 val_260 2 +262 val_262 262 val_262 2 +263 val_263 263 val_263 2 +265 val_265 265 val_265 2 +266 val_266 266 val_266 2 +27 val_27 NULL NULL 2 +27 val_27 27 val_27 2 +272 val_272 272 val_272 2 +273 val_273 273 val_273 2 +274 val_274 274 val_274 2 +275 val_275 275 val_275 2 +277 val_277 277 val_277 2 +278 val_278 278 val_278 2 +28 val_28 NULL NULL 2 +28 val_28 28 val_28 2 +280 val_280 280 val_280 2 +281 val_281 281 val_281 2 +282 val_282 282 val_282 2 +283 val_283 283 val_283 2 +284 val_284 284 val_284 2 +285 val_285 285 val_285 2 +286 val_286 286 val_286 2 +287 val_287 287 val_287 2 +288 val_288 288 val_288 2 +289 val_289 289 val_289 2 +291 val_291 291 val_291 2 +292 val_292 292 val_292 2 +296 val_296 296 val_296 2 +298 val_298 298 val_298 2 +30 val_30 NULL NULL 2 +30 val_30 30 val_30 2 +302 val_302 302 val_302 2 +305 val_305 305 val_305 2 +306 val_306 306 val_306 2 +307 val_307 307 val_307 2 +308 val_308 308 val_308 2 +309 val_309 309 val_309 2 +310 val_310 310 val_310 2 +311 val_311 311 val_311 2 +315 val_315 315 val_315 2 +316 val_316 316 val_316 2 +317 val_317 317 val_317 2 +318 val_318 318 val_318 2 +321 val_321 321 val_321 2 +322 val_322 322 val_322 2 +323 val_323 323 val_323 2 +325 val_325 325 val_325 2 +327 val_327 327 val_327 2 +33 val_33 NULL NULL 2 +33 val_33 33 val_33 2 +331 val_331 331 val_331 2 +332 val_332 332 val_332 2 +333 val_333 333 val_333 2 +335 val_335 335 val_335 2 +336 val_336 336 val_336 2 +338 val_338 338 val_338 2 +339 val_339 339 val_339 2 +34 val_34 NULL NULL 2 +34 val_34 34 val_34 2 +341 val_341 341 val_341 2 +342 val_342 342 val_342 2 +344 val_344 344 val_344 2 +345 val_345 345 val_345 2 +348 val_348 348 val_348 2 +35 val_35 NULL NULL 2 +35 val_35 35 val_35 2 +351 val_351 351 val_351 2 +353 val_353 353 val_353 2 +356 val_356 356 val_356 2 +360 val_360 360 val_360 2 +362 val_362 362 val_362 2 +364 val_364 364 val_364 2 +365 val_365 365 val_365 2 +366 val_366 366 val_366 2 +367 val_367 367 val_367 2 +368 val_368 368 val_368 2 +369 val_369 369 val_369 2 +37 val_37 NULL NULL 2 +37 val_37 37 val_37 2 +373 val_373 373 val_373 2 +374 val_374 374 val_374 2 +375 val_375 375 val_375 2 +377 val_377 377 val_377 2 +378 val_378 378 val_378 2 +379 val_379 379 val_379 2 +382 val_382 382 val_382 2 +384 val_384 384 val_384 2 +386 val_386 386 val_386 2 +389 val_389 389 val_389 2 +392 val_392 392 val_392 2 +393 val_393 393 val_393 2 +394 val_394 394 val_394 2 +395 val_395 395 val_395 2 +396 val_396 396 val_396 2 +397 val_397 397 val_397 2 +399 val_399 399 val_399 2 +4 val_4 4 val_4 2 +400 val_400 400 val_400 2 +401 val_401 401 val_401 2 +402 val_402 402 val_402 2 +403 val_403 403 val_403 2 +404 val_404 404 val_404 2 +406 val_406 406 val_406 2 +407 val_407 407 val_407 2 +409 val_409 409 val_409 2 +41 val_41 NULL NULL 2 +41 val_41 41 val_41 2 +411 val_411 411 val_411 2 +413 val_413 413 val_413 2 +414 val_414 414 val_414 2 +417 val_417 417 val_417 2 +418 val_418 418 val_418 2 +419 val_419 419 val_419 2 +42 val_42 NULL NULL 2 +42 val_42 42 val_42 2 +421 val_421 421 val_421 2 +424 val_424 424 val_424 2 +427 val_427 427 val_427 2 +429 val_429 429 val_429 2 +43 val_43 NULL NULL 2 +43 val_43 43 val_43 2 +430 val_430 430 val_430 2 +431 val_431 431 val_431 2 +432 val_432 432 val_432 2 +435 val_435 435 val_435 2 +436 val_436 436 val_436 2 +437 val_437 437 val_437 2 +438 val_438 438 val_438 2 +439 val_439 439 val_439 2 +44 val_44 NULL NULL 2 +44 val_44 44 val_44 2 +443 val_443 443 val_443 2 +444 val_444 444 val_444 2 +446 val_446 446 val_446 2 +448 val_448 448 val_448 2 +449 val_449 449 val_449 2 +452 val_452 452 val_452 2 +453 val_453 453 val_453 2 +454 val_454 454 val_454 2 +455 val_455 455 val_455 2 +457 val_457 457 val_457 2 +458 val_458 458 val_458 2 +459 val_459 459 val_459 2 +460 val_460 460 val_460 2 +462 val_462 462 val_462 2 +463 val_463 463 val_463 2 +466 val_466 466 val_466 2 +467 val_467 467 val_467 2 +468 val_468 468 val_468 2 +469 val_469 469 val_469 2 +47 val_47 NULL NULL 2 +47 val_47 47 val_47 2 +470 val_470 470 val_470 2 +472 val_472 472 val_472 2 +475 val_475 475 val_475 2 +477 val_477 477 val_477 2 +478 val_478 478 val_478 2 +479 val_479 479 val_479 2 +480 val_480 480 val_480 2 +481 val_481 481 val_481 2 +482 val_482 482 val_482 2 +483 val_483 483 val_483 2 +484 val_484 484 val_484 2 +485 val_485 485 val_485 2 +487 val_487 487 val_487 2 +489 val_489 489 val_489 2 +490 val_490 490 val_490 2 +491 val_491 491 val_491 2 +492 val_492 492 val_492 2 +493 val_493 493 val_493 2 +494 val_494 494 val_494 2 +495 val_495 495 val_495 2 +496 val_496 496 val_496 2 +497 val_497 497 val_497 2 +498 val_498 498 val_498 2 +5 val_5 5 val_5 2 +51 val_51 51 val_51 2 +53 val_53 53 val_53 2 +54 val_54 54 val_54 2 +57 val_57 57 val_57 2 +58 val_58 58 val_58 2 +64 val_64 64 val_64 2 +65 val_65 65 val_65 2 +66 val_66 66 val_66 2 +67 val_67 67 val_67 2 +69 val_69 69 val_69 2 +70 val_70 70 val_70 2 +72 val_72 72 val_72 2 +74 val_74 74 val_74 2 +76 val_76 76 val_76 2 +77 val_77 77 val_77 2 +78 val_78 78 val_78 2 +8 val_8 8 val_8 2 +80 val_80 80 val_80 2 +82 val_82 82 val_82 2 +83 val_83 83 val_83 2 +84 val_84 84 val_84 2 +85 val_85 85 val_85 2 +86 val_86 86 val_86 2 +87 val_87 87 val_87 2 +9 val_9 9 val_9 2 +90 val_90 90 val_90 2 +92 val_92 92 val_92 2 +95 val_95 95 val_95 2 +96 val_96 96 val_96 2 +97 val_97 97 val_97 2 +98 val_98 98 val_98 2 +PREHOOK: query: -- union23.q + +explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +PREHOOK: type: QUERY +POSTHOOK: query: -- union23.q + +explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union24.q + +-- SORT_QUERY_RESULTS + +create table src2 as select key, count(1) as count from src group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src2 +POSTHOOK: query: -- union24.q + +-- SORT_QUERY_RESULTS + +create table src2 as select key, count(1) as count from src group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src2 +PREHOOK: query: create table src3 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src3 +POSTHOOK: query: create table src3 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src3 +PREHOOK: query: create table src4 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src4 +POSTHOOK: query: create table src4 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src4 +PREHOOK: query: create table src5 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src4 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src5 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_FUNCTION + count + 1 + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_GROUPBY + TOK_TABLE_OR_COL + key + _u3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 8 <- Union 2 (CONTAINS) + Map 9 <- Union 4 (CONTAINS) + Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 10 + Map Operator Tree: + TableScan + alias: src5 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [src5] + Map 8 + Map Operator Tree: + TableScan + alias: src3 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 9 + Map Operator Tree: + TableScan + alias: src4 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [src4] + Reducer 11 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Reducer 3 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Reducer 5 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Reducer 7 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 1 +0 3 +2 1 +4 1 +5 1 +5 3 +8 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src4 + a + TOK_TABREF + TOK_TABNAME + src5 + b + = + . + TOK_TABLE_OR_COL + a + key + . + TOK_TABLE_OR_COL + b + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + key + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + count + count + TOK_WHERE + < + . + TOK_TABLE_OR_COL + a + key + 10 + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Map 8 (BROADCAST_EDGE), Union 4 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 6 + Map Operator Tree: + TableScan + alias: src3 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 7 + Map Operator Tree: + TableScan + alias: a + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Map 8 => 103 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col6 + input vertices: + 1 Map 8 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [a] + Map 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: count (type: bigint) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [b] + Reducer 3 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Reducer 5 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src4 + a + TOK_TABREF + TOK_TABNAME + src5 + b + = + . + TOK_TABLE_OR_COL + a + key + . + TOK_TABLE_OR_COL + b + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + key + key + TOK_SELEXPR + TOK_FUNCTION + count + 1 + count + TOK_WHERE + < + . + TOK_TABLE_OR_COL + a + key + 10 + TOK_GROUPBY + . + TOK_TABLE_OR_COL + a + key + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Map 9 (BROADCAST_EDGE) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 + Truncated Path -> Alias: + /src2 [src2] + Map 6 + Map Operator Tree: + TableScan + alias: src3 + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src3 [src3] + Map 7 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Map 9 => 103 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 9 + Position of Big Table: 0 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [a] + Map 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [b] + Reducer 3 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Reducer 5 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Reducer 8 + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + tag: -1 + auto parallelism: true + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 1 +0 3 +2 1 +4 1 +5 1 +5 3 +8 1 +9 1 +PREHOOK: query: -- union25.q + +create table tmp_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: -- union25.q + +create table tmp_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +POSTHOOK: type: CREATETABLE_AS_SELECT +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 5 (CONTAINS) + Map 7 <- Union 5 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create Table Operator: + Create Table + columns: counts bigint, key string, value string + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: -- union26.q + +-- SORT_QUERY_RESULTS + +EXPLAIN +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +POSTHOOK: query: -- union26.q + +-- SORT_QUERY_RESULTS + +EXPLAIN +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 4 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: srcpart + Lateral View Forward + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: -- union27.q + +-- SORT_BEFORE_DIFF +create table jackson_sev_same as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_same +POSTHOOK: query: -- union27.q + +-- SORT_BEFORE_DIFF +create table jackson_sev_same as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_same +PREHOOK: query: create table dim_pho as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dim_pho +POSTHOOK: query: create table dim_pho as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dim_pho +PREHOOK: query: create table jackson_sev_add as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_add +POSTHOOK: query: create table jackson_sev_add as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_add +PREHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +POSTHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Map 5 <- Reducer 3 (BROADCAST_EDGE) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dim_pho + Filter Operator + predicate: (key = 97) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Select Operator + expressions: '97' (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: jackson_sev_add + Filter Operator + predicate: (key = 97) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Select Operator + expressions: '97' (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '97' (type: string) + 1 '97' (type: string) + outputColumnNames: _col6 + input vertices: + 1 Reducer 3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '97' (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '97' (type: string) + sort order: + + Map-reduce partition columns: '97' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +PREHOOK: Input: default@dim_pho +PREHOOK: Input: default@jackson_sev_add +PREHOOK: Input: default@jackson_sev_same +#### A masked pattern was here #### +POSTHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dim_pho +POSTHOOK: Input: default@jackson_sev_add +POSTHOOK: Input: default@jackson_sev_same +#### A masked pattern was here #### +97 val_97 +97 val_97 +PREHOOK: query: -- union28.q + +create table union_subq_union(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: -- union28.q + +create table union_subq_union(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union +PREHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 2 (CONTAINS), Union 6 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (SIMPLE_EDGE), Union 6 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Map 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 9 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Union 2 + Vertex: Union 2 + Union 6 + Vertex: Union 6 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union +POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union29.q + +create table union_subq_union29(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: -- union29.q + +create table union_subq_union29(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union29 +PREHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 5 (CONTAINS) + Map 7 <- Union 5 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union29 +POSTHOOK: Lineage: union_subq_union29.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union29.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union29 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union29 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union29 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union29 +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union3.q + +-- SORT_BEFORE_DIFF + +explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +PREHOOK: type: QUERY +POSTHOOK: query: -- union3.q + +-- SORT_BEFORE_DIFF + +explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 14 <- Map 13 (SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 8 <- Union 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Map 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Map 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reducer 10 + Reduce Operator Tree: + Limit + Number of rows: 1 + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 12 + Reduce Operator Tree: + Limit + Number of rows: 1 + Select Operator + expressions: 3 (type: int) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 14 + Reduce Operator Tree: + Limit + Number of rows: 1 + Select Operator + expressions: 4 (type: int) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 2 + Reduce Operator Tree: + Limit + Number of rows: 1 + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reducer 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE union_out (id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_out +POSTHOOK: query: CREATE TABLE union_out (id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_out +PREHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_out +POSTHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_out +POSTHOOK: Lineage: union_out.id EXPRESSION [] +PREHOOK: query: select * from union_out +PREHOOK: type: QUERY +PREHOOK: Input: default@union_out +#### A masked pattern was here #### +POSTHOOK: query: select * from union_out +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_out +#### A masked pattern was here #### +1 +2 +3 +4 +PREHOOK: query: -- union30.q + +create table union_subq_union30(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: -- union30.q + +create table union_subq_union30(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union30 +PREHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 12 <- Union 4 (CONTAINS) + Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 2 (CONTAINS), Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 10 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Map 12 + Map Operator Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reducer 11 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union30 + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Reducer 9 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 8 + Vertex: Union 8 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union30 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union30 +POSTHOOK: Lineage: union_subq_union30.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union30.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union30 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union30 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union30 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union30 +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union31.q + +-- SORT_QUERY_RESULTS + +drop table t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- union31.q + +-- SORT_QUERY_RESULTS + +drop table t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table t2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t2 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t3(key string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3 +POSTHOOK: query: create table t3(key string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3 +PREHOOK: query: create table t4(value string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t4 +POSTHOOK: query: create table t4(value string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4 +PREHOOK: query: explain +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 + Union 2 + Vertex: Union 2 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t3 +PREHOOK: Output: default@t4 +POSTHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t3 +POSTHOOK: Output: default@t4 +POSTHOOK: Lineage: t3.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t3.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t4.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select * from t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t4 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +POSTHOOK: query: select * from t4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4 +#### A masked pattern was here #### +val_0 1 +val_2 1 +val_4 1 +val_5 1 +val_8 1 +val_9 1 +PREHOOK: query: create table t5(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t5 +POSTHOOK: query: create table t5(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t5 +PREHOOK: query: create table t6(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t6 +POSTHOOK: query: create table t6(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t6 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Map 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 5 + Reduce Operator Tree: + Forward + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t5 +PREHOOK: Output: default@t6 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t5 +POSTHOOK: Output: default@t6 +POSTHOOK: Lineage: t5.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t5.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t6.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t6.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +PREHOOK: query: select * from t5 +PREHOOK: type: QUERY +PREHOOK: Input: default@t5 +#### A masked pattern was here #### +POSTHOOK: query: select * from t5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t5 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: select * from t6 +PREHOOK: type: QUERY +PREHOOK: Input: default@t6 +#### A masked pattern was here #### +POSTHOOK: query: select * from t6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t6 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: drop table t2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t2 +POSTHOOK: query: drop table t2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t1 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t7(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t7 +POSTHOOK: query: create table t7(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t7 +PREHOOK: query: create table t8(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez + Edges: + Map 6 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Map 6 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: key (type: string), cnt (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reducer 5 + Reduce Operator Tree: + Forward + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7 + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 + Union 3 + Vertex: Union 3 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t7 +PREHOOK: Output: default@t8 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t7 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t7.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t7.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t8.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t8.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +PREHOOK: query: select * from t7 +PREHOOK: type: QUERY +PREHOOK: Input: default@t7 +#### A masked pattern was here #### +POSTHOOK: query: select * from t7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t7 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t8 +PREHOOK: type: QUERY +PREHOOK: Input: default@t8 +#### A masked pattern was here #### +POSTHOOK: query: select * from t8 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: -- union32.q + +-- SORT_QUERY_RESULTS + +-- This tests various union queries which have columns on one side of the query +-- being of double type and those on the other side another + +drop table if exists t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: -- union32.q + +-- SORT_QUERY_RESULTS + +-- This tests various union queries which have columns on one side of the query +-- being of double type and those on the other side another + +drop table if exists t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: drop table if exists t2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t2 +POSTHOOK: query: drop table if exists t2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t2 +PREHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: -- Test simple union with double +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +ORDER BY key +PREHOOK: type: QUERY +POSTHOOK: query: -- Test simple union with double +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +ORDER BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Map 5 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: UDFToDouble(UDFToLong(key)) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the left +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the left +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 4 + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the right +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the right +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Map 4 + Map Operator Tree: + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 5 + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Map 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the left selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the left selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + input vertices: + 1 Map 4 + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(UDFToDouble(_col5)) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: UDFToDouble(key) (type: double), key (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0 +0.0 0.0 +2.0 2 +2.0 2.0 +4.0 4 +4.0 4.0 +5.0 5 +5.0 5.0 +8.0 8 +8.0 8.0 +9.0 9 +9.0 9.0 +PREHOOK: query: -- Test union with join on the right selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the right selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2 + Select Operator + expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: double), _col1 (type: double) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Map 4 + Map Operator Tree: + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + input vertices: + 1 Map 5 + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToDouble(_col5) (type: double) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: double), _col1 (type: double) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Map 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0.0 +2.0 2.0 +4.0 4.0 +5.0 5.0 +8.0 8.0 +9.0 9.0 +PREHOOK: query: -- union33.q + +-- SORT_BEFORE_DIFF +-- This tests that a UNION DISTINCT with a map only subquery on one side and a +-- subquery involving two map reduce jobs on the other runs correctly. + +drop table if exists test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- union33.q + +-- SORT_BEFORE_DIFF +-- This tests that a UNION DISTINCT with a map only subquery on one side and a +-- subquery involving two map reduce jobs on the other runs correctly. + +drop table if exists test_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_src +POSTHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_src +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Select Operator + expressions: '0' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 2 + Vertex: Union 2 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: src + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Select Operator + expressions: '0' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_1 +POSTHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_1 +PREHOOK: query: create table src10_2 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_2 +POSTHOOK: query: create table src10_2 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_2 +PREHOOK: query: create table src10_3 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_3 +POSTHOOK: query: create table src10_3 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_3 +PREHOOK: query: create table src10_4 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_4 +POSTHOOK: query: create table src10_4 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_4 +PREHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src10_1 +PREHOOK: Output: default@src10_2 +PREHOOK: Output: default@src10_3 +PREHOOK: Output: default@src10_4 +POSTHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src10_1 +POSTHOOK: Output: default@src10_2 +POSTHOOK: Output: default@src10_3 +POSTHOOK: Output: default@src10_4 +POSTHOOK: Lineage: src10_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, +-- we can use a single MR job to evaluate this entire query. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +POSTHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, +-- we can use a single MR job to evaluate this entire query. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 6 <- Union 7 (CONTAINS) + Map 9 <- Union 7 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 8 <- Union 2 (CONTAINS), Union 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src10_1 + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 5 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: src10_2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: src10_3 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 9 + Map Operator Tree: + TableScan + alias: src10_4 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 2 + Vertex: Union 2 + Union 7 + Vertex: Union 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 +#### A masked pattern was here #### +165 val_165 +238 val_238 +255 val_255 +27 val_27 +278 val_278 +311 val_311 +409 val_409 +484 val_484 +86 val_86 +98 val_98 +PREHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, +-- we need to use two MR jobs to evaluate this query. +-- The first job is for the Join of sub1 and sub2. The second job +-- is for the UNION DISTINCT and ORDER BY. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +POSTHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, +-- we need to use two MR jobs to evaluate this query. +-- The first job is for the Join of sub1 and sub2. The second job +-- is for the UNION DISTINCT and ORDER BY. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 10 <- Union 8 (CONTAINS) + Map 7 <- Union 8 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 9 <- Union 3 (CONTAINS), Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src10_1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Map 10 + Map Operator Tree: + TableScan + alias: src10_4 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: src10_2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map 7 + Map Operator Tree: + TableScan + alias: src10_3 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 9 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Union 3 + Vertex: Union 3 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 +#### A masked pattern was here #### +165 val_165 +238 val_238 +255 val_255 +27 val_27 +278 val_278 +311 val_311 +409 val_409 +484 val_484 +86 val_86 +98 val_98 +PREHOOK: query: -- union4.q + + + +-- union case: both subqueries are map-reduce jobs on same input, followed by filesink + +drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union4.q + + + +-- union case: both subqueries are map-reduce jobs on same input, followed by filesink + +drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: create table tmptable(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [] +POSTHOOK: Lineage: tmptable.value EXPRESSION [] +PREHOOK: query: select * from tmptable x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### +tst1 500 +tst2 500 +PREHOOK: query: -- union5.q + + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union5.q + + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +PREHOOK: query: -- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +tst1 500 +PREHOOK: query: -- union7.q + + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union7.q + + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union8.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union8.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Union 4 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union9.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union9.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 7 <- Union 2 (CONTAINS) + Map 8 <- Union 4 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map 8 + Map Operator Tree: + TableScan + alias: s1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Reducer 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out new file mode 100644 index 0000000..d356f8d --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out @@ -0,0 +1,410 @@ +PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u1 +POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u1 +PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u2 +POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u2 +PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u3 +POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u3 +PREHOOK: query: select * from u1 +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +#### A masked pattern was here #### +POSTHOOK: query: select * from u1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: select * from u2 +PREHOOK: type: QUERY +PREHOOK: Input: default@u2 +#### A masked pattern was here #### +POSTHOOK: query: select * from u2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u2 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +PREHOOK: query: select * from u3 +PREHOOK: type: QUERY +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select * from u3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +0 val_0 +0 val_0 +0 val_0 +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union all +select key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union all +select key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union distinct +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union distinct +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union +select key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union +select key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: describe extended v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: describe extended v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +key string +value string + +#### A masked pattern was here #### +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from +( +select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` +union +select `u2`.`key`, `u2`.`value` from `default`.`u2` +union all +select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` +) `tab`, tableType:VIRTUAL_VIEW) +PREHOOK: query: select * from v +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +POSTHOOK: Input: default@v +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 diff --git a/ql/src/test/results/clientpositive/union22.q.out b/ql/src/test/results/clientpositive/union22.q.out index 672fa77..9115287 100644 --- a/ql/src/test/results/clientpositive/union22.q.out +++ b/ql/src/test/results/clientpositive/union22.q.out @@ -83,7 +83,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/union24.q.out b/ql/src/test/results/clientpositive/union24.q.out index 0657b65..2eb709b 100644 --- a/ql/src/test/results/clientpositive/union24.q.out +++ b/ql/src/test/results/clientpositive/union24.q.out @@ -69,9 +69,9 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION - TOK_UNION - TOK_UNION + TOK_UNIONALL + TOK_UNIONALL + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -682,8 +682,8 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION - TOK_UNION + TOK_UNIONALL + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF @@ -1253,8 +1253,8 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION - TOK_UNION + TOK_UNIONALL + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/union3.q.out b/ql/src/test/results/clientpositive/union3.q.out index 810d3af..b956d88 100644 --- a/ql/src/test/results/clientpositive/union3.q.out +++ b/ql/src/test/results/clientpositive/union3.q.out @@ -5,17 +5,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY POSTHOOK: query: -- SORT_BEFORE_DIFF @@ -25,28 +24,25 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2, Stage-5, Stage-6, Stage-7 + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4, Stage-5 + Stage-3 is a root stage Stage-4 is a root stage - Stage-5 depends on stages: Stage-4 - Stage-6 is a root stage - Stage-7 is a root stage - Stage-0 depends on stages: Stage-3 + Stage-5 is a root stage + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -67,95 +63,66 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce Map Operator Tree: TableScan - Reduce Output Operator - key expressions: 1 (type: int) - sort order: + - Map-reduce partition columns: 1 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan Union Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-4 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -173,35 +140,18 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: 2 (type: int) - sort order: + - Map-reduce partition columns: 2 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-6 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -230,7 +180,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-7 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -278,17 +228,16 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -298,27 +247,26 @@ SELECT * FROM ( SELECT 1 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 2 AS id FROM (SELECT * FROM src LIMIT 1) s1 - CLUSTER BY id UNION ALL SELECT 3 AS id FROM (SELECT * FROM src LIMIT 1) s2 UNION ALL SELECT 4 AS id FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_out POSTHOOK: Lineage: union_out.id EXPRESSION [] -PREHOOK: query: select * from union_out cluster by id +PREHOOK: query: select * from union_out PREHOOK: type: QUERY PREHOOK: Input: default@union_out #### A masked pattern was here #### -POSTHOOK: query: select * from union_out cluster by id +POSTHOOK: query: select * from union_out POSTHOOK: type: QUERY POSTHOOK: Input: default@union_out #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/unionDistinct_1.q.out new file mode 100644 index 0000000..1f672a4 --- /dev/null +++ b/ql/src/test/results/clientpositive/unionDistinct_1.q.out @@ -0,0 +1,17314 @@ +PREHOOK: query: -- union10.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + +create table tmptable(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union10.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by filesink + +create table tmptable(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-5 is a root stage + Stage-6 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [] +POSTHOOK: Lineage: tmptable.value EXPRESSION [] +PREHOOK: query: select * from tmptable x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### +tst1 500 +tst2 500 +tst3 500 +PREHOOK: query: -- union11.q + + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union11.q + + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-3 depends on stages: Stage-2, Stage-5 + Stage-4 is a root stage + Stage-5 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +tst3 1 +PREHOOK: query: -- union12.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink + +create table tmptable12(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable12 +POSTHOOK: query: -- union12.q + + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, different inputs for all sub-queries, followed by filesink + +create table tmptable12(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable12 +PREHOOK: query: explain +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-5 is a root stage + Stage-6 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable12 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable12 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@tmptable12 +POSTHOOK: query: insert overwrite table tmptable12 + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src1 s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from srcbucket s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@tmptable12 +POSTHOOK: Lineage: tmptable12.key EXPRESSION [] +POSTHOOK: Lineage: tmptable12.value EXPRESSION [] +PREHOOK: query: select * from tmptable12 x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable12 +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable12 x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable12 +#### A masked pattern was here #### +tst1 500 +tst2 25 +tst3 1000 +PREHOOK: query: -- union13.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are a map-only jobs, same input, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union13.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are a map-only jobs, same input, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union14.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union14.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union15.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union15.q + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union16.q + +-- SORT_BEFORE_DIFF +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +POSTHOOK: query: -- union16.q + +-- SORT_BEFORE_DIFF +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-5 + Stage-7 depends on stages: Stage-6 + Stage-8 depends on stages: Stage-7 + Stage-9 depends on stages: Stage-8 + Stage-10 depends on stages: Stage-9 + Stage-11 depends on stages: Stage-10 + Stage-12 depends on stages: Stage-11 + Stage-13 depends on stages: Stage-12 + Stage-14 depends on stages: Stage-13 + Stage-15 depends on stages: Stage-14 + Stage-16 depends on stages: Stage-15 + Stage-17 depends on stages: Stage-16 + Stage-18 depends on stages: Stage-17 + Stage-19 depends on stages: Stage-18 + Stage-20 depends on stages: Stage-19 + Stage-21 depends on stages: Stage-20 + Stage-22 depends on stages: Stage-21 + Stage-23 depends on stages: Stage-22 + Stage-24 depends on stages: Stage-23 + Stage-25 depends on stages: Stage-24 + Stage-0 depends on stages: Stage-25 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-9 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-11 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-12 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-13 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-14 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-15 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-16 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-17 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-18 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-19 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-20 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-21 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-22 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-23 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-24 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-25 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: -- union17.q + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST1 +POSTHOOK: query: -- union17.q + +CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST1 +PREHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST2 +POSTHOOK: query: CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST2 +PREHOOK: query: -- SORT_BEFORE_DIFF +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_BEFORE_DIFF +-- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-5 + Stage-6 depends on stages: Stage-0 + Stage-7 depends on stages: Stage-4 + Stage-1 depends on stages: Stage-7 + Stage-8 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: substr(_col1, 5) (type: string) + sort order: + + Map-reduce partition columns: substr(_col1, 5) (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Forward + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(DISTINCT KEY._col0) + keys: VALUE._col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + aggregations: count(DISTINCT KEY._col0) + keys: VALUE._col0 (type: string), VALUE._col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: final + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-6 + Stats-Aggr Operator + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 + + Stage: Stage-8 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1 +PREHOOK: Output: default@dest2 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key +INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1 +POSTHOOK: Output: default@dest2 +POSTHOOK: Lineage: dest1.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest2.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST1.* FROM DEST1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST1.* FROM DEST1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST2.* FROM DEST2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST2.* FROM DEST2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest2 +#### A masked pattern was here #### +0 val_0 1 +10 val_10 1 +100 val_100 1 +103 val_103 1 +104 val_104 1 +105 val_105 1 +11 val_11 1 +111 val_111 1 +113 val_113 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +119 val_119 1 +12 val_12 1 +120 val_120 1 +125 val_125 1 +126 val_126 1 +128 val_128 1 +129 val_129 1 +131 val_131 1 +133 val_133 1 +134 val_134 1 +136 val_136 1 +137 val_137 1 +138 val_138 1 +143 val_143 1 +145 val_145 1 +146 val_146 1 +149 val_149 1 +15 val_15 1 +150 val_150 1 +152 val_152 1 +153 val_153 1 +155 val_155 1 +156 val_156 1 +157 val_157 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +163 val_163 1 +164 val_164 1 +165 val_165 1 +166 val_166 1 +167 val_167 1 +168 val_168 1 +169 val_169 1 +17 val_17 1 +170 val_170 1 +172 val_172 1 +174 val_174 1 +175 val_175 1 +176 val_176 1 +177 val_177 1 +178 val_178 1 +179 val_179 1 +18 val_18 1 +180 val_180 1 +181 val_181 1 +183 val_183 1 +186 val_186 1 +187 val_187 1 +189 val_189 1 +19 val_19 1 +190 val_190 1 +191 val_191 1 +192 val_192 1 +193 val_193 1 +194 val_194 1 +195 val_195 1 +196 val_196 1 +197 val_197 1 +199 val_199 1 +2 val_2 1 +20 val_20 1 +200 val_200 1 +201 val_201 1 +202 val_202 1 +203 val_203 1 +205 val_205 1 +207 val_207 1 +208 val_208 1 +209 val_209 1 +213 val_213 1 +214 val_214 1 +216 val_216 1 +217 val_217 1 +218 val_218 1 +219 val_219 1 +221 val_221 1 +222 val_222 1 +223 val_223 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +229 val_229 1 +230 val_230 1 +233 val_233 1 +235 val_235 1 +237 val_237 1 +238 val_238 1 +239 val_239 1 +24 val_24 1 +241 val_241 1 +242 val_242 1 +244 val_244 1 +247 val_247 1 +248 val_248 1 +249 val_249 1 +252 val_252 1 +255 val_255 1 +256 val_256 1 +257 val_257 1 +258 val_258 1 +26 val_26 1 +260 val_260 1 +262 val_262 1 +263 val_263 1 +265 val_265 1 +266 val_266 1 +27 val_27 1 +272 val_272 1 +273 val_273 1 +274 val_274 1 +275 val_275 1 +277 val_277 1 +278 val_278 1 +28 val_28 1 +280 val_280 1 +281 val_281 1 +282 val_282 1 +283 val_283 1 +284 val_284 1 +285 val_285 1 +286 val_286 1 +287 val_287 1 +288 val_288 1 +289 val_289 1 +291 val_291 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +30 val_30 1 +302 val_302 1 +305 val_305 1 +306 val_306 1 +307 val_307 1 +308 val_308 1 +309 val_309 1 +310 val_310 1 +311 val_311 1 +315 val_315 1 +316 val_316 1 +317 val_317 1 +318 val_318 1 +321 val_321 1 +322 val_322 1 +323 val_323 1 +325 val_325 1 +327 val_327 1 +33 val_33 1 +331 val_331 1 +332 val_332 1 +333 val_333 1 +335 val_335 1 +336 val_336 1 +338 val_338 1 +339 val_339 1 +34 val_34 1 +341 val_341 1 +342 val_342 1 +344 val_344 1 +345 val_345 1 +348 val_348 1 +35 val_35 1 +351 val_351 1 +353 val_353 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +365 val_365 1 +366 val_366 1 +367 val_367 1 +368 val_368 1 +369 val_369 1 +37 val_37 1 +373 val_373 1 +374 val_374 1 +375 val_375 1 +377 val_377 1 +378 val_378 1 +379 val_379 1 +382 val_382 1 +384 val_384 1 +386 val_386 1 +389 val_389 1 +392 val_392 1 +393 val_393 1 +394 val_394 1 +395 val_395 1 +396 val_396 1 +397 val_397 1 +399 val_399 1 +4 val_4 1 +400 val_400 1 +401 val_401 1 +402 val_402 1 +403 val_403 1 +404 val_404 1 +406 val_406 1 +407 val_407 1 +409 val_409 1 +41 val_41 1 +411 val_411 1 +413 val_413 1 +414 val_414 1 +417 val_417 1 +418 val_418 1 +419 val_419 1 +42 val_42 1 +421 val_421 1 +424 val_424 1 +427 val_427 1 +429 val_429 1 +43 val_43 1 +430 val_430 1 +431 val_431 1 +432 val_432 1 +435 val_435 1 +436 val_436 1 +437 val_437 1 +438 val_438 1 +439 val_439 1 +44 val_44 1 +443 val_443 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +449 val_449 1 +452 val_452 1 +453 val_453 1 +454 val_454 1 +455 val_455 1 +457 val_457 1 +458 val_458 1 +459 val_459 1 +460 val_460 1 +462 val_462 1 +463 val_463 1 +466 val_466 1 +467 val_467 1 +468 val_468 1 +469 val_469 1 +47 val_47 1 +470 val_470 1 +472 val_472 1 +475 val_475 1 +477 val_477 1 +478 val_478 1 +479 val_479 1 +480 val_480 1 +481 val_481 1 +482 val_482 1 +483 val_483 1 +484 val_484 1 +485 val_485 1 +487 val_487 1 +489 val_489 1 +490 val_490 1 +491 val_491 1 +492 val_492 1 +493 val_493 1 +494 val_494 1 +495 val_495 1 +496 val_496 1 +497 val_497 1 +498 val_498 1 +5 val_5 1 +51 val_51 1 +53 val_53 1 +54 val_54 1 +57 val_57 1 +58 val_58 1 +64 val_64 1 +65 val_65 1 +66 val_66 1 +67 val_67 1 +69 val_69 1 +70 val_70 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +77 val_77 1 +78 val_78 1 +8 val_8 1 +80 val_80 1 +82 val_82 1 +83 val_83 1 +84 val_84 1 +85 val_85 1 +86 val_86 1 +87 val_87 1 +9 val_9 1 +90 val_90 1 +92 val_92 1 +95 val_95 1 +96 val_96 1 +97 val_97 1 +98 val_98 1 +tst1 500 1 +PREHOOK: query: -- union18.q + +CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST118 +POSTHOOK: query: -- union18.q + +CREATE TABLE DEST118(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST118 +PREHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST218 +POSTHOOK: query: CREATE TABLE DEST218(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST218 +PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest118 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest218 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest118 +PREHOOK: Output: default@dest218 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST118 SELECT unionsrc.key, unionsrc.value +INSERT OVERWRITE TABLE DEST218 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest118 +POSTHOOK: Output: default@dest218 +POSTHOOK: Lineage: dest118.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest118.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest218.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest118 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST118.* FROM DEST118 SORT BY DEST118.key, DEST118.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest118 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +tst1 500 +PREHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest218 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST218.* FROM DEST218 SORT BY DEST218.key, DEST218.val1, DEST218.val2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest218 +#### A masked pattern was here #### +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: -- union19.q + + + + +CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST119 +POSTHOOK: query: -- union19.q + + + + +CREATE TABLE DEST119(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST119 +PREHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DEST219 +POSTHOOK: query: CREATE TABLE DEST219(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DEST219 +PREHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +POSTHOOK: query: -- union case:map-reduce sub-queries followed by multi-table insert + +explain +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest119 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest119 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest219 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest119 +PREHOOK: Output: default@dest219 +POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +INSERT OVERWRITE TABLE DEST119 SELECT unionsrc.key, count(unionsrc.value) group by unionsrc.key +INSERT OVERWRITE TABLE DEST219 SELECT unionsrc.key, unionsrc.value, unionsrc.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest119 +POSTHOOK: Output: default@dest219 +POSTHOOK: Lineage: dest119.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest119.value EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.key EXPRESSION [(src)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val1 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest219.val2 EXPRESSION [(src)s1.null, (src)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest119 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest119 +#### A masked pattern was here #### +0 1 +10 1 +100 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +183 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +205 1 +207 1 +208 1 +209 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +244 1 +247 1 +248 1 +249 1 +252 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +262 1 +263 1 +265 1 +266 1 +27 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +291 1 +292 1 +296 1 +298 1 +30 1 +302 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +356 1 +360 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +382 1 +384 1 +386 1 +389 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +443 1 +444 1 +446 1 +448 1 +449 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +460 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +472 1 +475 1 +477 1 +478 1 +479 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +51 1 +53 1 +54 1 +57 1 +58 1 +64 1 +65 1 +66 1 +67 1 +69 1 +70 1 +72 1 +74 1 +76 1 +77 1 +78 1 +8 1 +80 1 +82 1 +83 1 +84 1 +85 1 +86 1 +87 1 +9 1 +90 1 +92 1 +95 1 +96 1 +97 1 +98 1 +tst1 1 +PREHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest219 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest219 +#### A masked pattern was here #### +0 val_0 val_0 +10 val_10 val_10 +100 val_100 val_100 +103 val_103 val_103 +104 val_104 val_104 +105 val_105 val_105 +11 val_11 val_11 +111 val_111 val_111 +113 val_113 val_113 +114 val_114 val_114 +116 val_116 val_116 +118 val_118 val_118 +119 val_119 val_119 +12 val_12 val_12 +120 val_120 val_120 +125 val_125 val_125 +126 val_126 val_126 +128 val_128 val_128 +129 val_129 val_129 +131 val_131 val_131 +133 val_133 val_133 +134 val_134 val_134 +136 val_136 val_136 +137 val_137 val_137 +138 val_138 val_138 +143 val_143 val_143 +145 val_145 val_145 +146 val_146 val_146 +149 val_149 val_149 +15 val_15 val_15 +150 val_150 val_150 +152 val_152 val_152 +153 val_153 val_153 +155 val_155 val_155 +156 val_156 val_156 +157 val_157 val_157 +158 val_158 val_158 +160 val_160 val_160 +162 val_162 val_162 +163 val_163 val_163 +164 val_164 val_164 +165 val_165 val_165 +166 val_166 val_166 +167 val_167 val_167 +168 val_168 val_168 +169 val_169 val_169 +17 val_17 val_17 +170 val_170 val_170 +172 val_172 val_172 +174 val_174 val_174 +175 val_175 val_175 +176 val_176 val_176 +177 val_177 val_177 +178 val_178 val_178 +179 val_179 val_179 +18 val_18 val_18 +180 val_180 val_180 +181 val_181 val_181 +183 val_183 val_183 +186 val_186 val_186 +187 val_187 val_187 +189 val_189 val_189 +19 val_19 val_19 +190 val_190 val_190 +191 val_191 val_191 +192 val_192 val_192 +193 val_193 val_193 +194 val_194 val_194 +195 val_195 val_195 +196 val_196 val_196 +197 val_197 val_197 +199 val_199 val_199 +2 val_2 val_2 +20 val_20 val_20 +200 val_200 val_200 +201 val_201 val_201 +202 val_202 val_202 +203 val_203 val_203 +205 val_205 val_205 +207 val_207 val_207 +208 val_208 val_208 +209 val_209 val_209 +213 val_213 val_213 +214 val_214 val_214 +216 val_216 val_216 +217 val_217 val_217 +218 val_218 val_218 +219 val_219 val_219 +221 val_221 val_221 +222 val_222 val_222 +223 val_223 val_223 +224 val_224 val_224 +226 val_226 val_226 +228 val_228 val_228 +229 val_229 val_229 +230 val_230 val_230 +233 val_233 val_233 +235 val_235 val_235 +237 val_237 val_237 +238 val_238 val_238 +239 val_239 val_239 +24 val_24 val_24 +241 val_241 val_241 +242 val_242 val_242 +244 val_244 val_244 +247 val_247 val_247 +248 val_248 val_248 +249 val_249 val_249 +252 val_252 val_252 +255 val_255 val_255 +256 val_256 val_256 +257 val_257 val_257 +258 val_258 val_258 +26 val_26 val_26 +260 val_260 val_260 +262 val_262 val_262 +263 val_263 val_263 +265 val_265 val_265 +266 val_266 val_266 +27 val_27 val_27 +272 val_272 val_272 +273 val_273 val_273 +274 val_274 val_274 +275 val_275 val_275 +277 val_277 val_277 +278 val_278 val_278 +28 val_28 val_28 +280 val_280 val_280 +281 val_281 val_281 +282 val_282 val_282 +283 val_283 val_283 +284 val_284 val_284 +285 val_285 val_285 +286 val_286 val_286 +287 val_287 val_287 +288 val_288 val_288 +289 val_289 val_289 +291 val_291 val_291 +292 val_292 val_292 +296 val_296 val_296 +298 val_298 val_298 +30 val_30 val_30 +302 val_302 val_302 +305 val_305 val_305 +306 val_306 val_306 +307 val_307 val_307 +308 val_308 val_308 +309 val_309 val_309 +310 val_310 val_310 +311 val_311 val_311 +315 val_315 val_315 +316 val_316 val_316 +317 val_317 val_317 +318 val_318 val_318 +321 val_321 val_321 +322 val_322 val_322 +323 val_323 val_323 +325 val_325 val_325 +327 val_327 val_327 +33 val_33 val_33 +331 val_331 val_331 +332 val_332 val_332 +333 val_333 val_333 +335 val_335 val_335 +336 val_336 val_336 +338 val_338 val_338 +339 val_339 val_339 +34 val_34 val_34 +341 val_341 val_341 +342 val_342 val_342 +344 val_344 val_344 +345 val_345 val_345 +348 val_348 val_348 +35 val_35 val_35 +351 val_351 val_351 +353 val_353 val_353 +356 val_356 val_356 +360 val_360 val_360 +362 val_362 val_362 +364 val_364 val_364 +365 val_365 val_365 +366 val_366 val_366 +367 val_367 val_367 +368 val_368 val_368 +369 val_369 val_369 +37 val_37 val_37 +373 val_373 val_373 +374 val_374 val_374 +375 val_375 val_375 +377 val_377 val_377 +378 val_378 val_378 +379 val_379 val_379 +382 val_382 val_382 +384 val_384 val_384 +386 val_386 val_386 +389 val_389 val_389 +392 val_392 val_392 +393 val_393 val_393 +394 val_394 val_394 +395 val_395 val_395 +396 val_396 val_396 +397 val_397 val_397 +399 val_399 val_399 +4 val_4 val_4 +400 val_400 val_400 +401 val_401 val_401 +402 val_402 val_402 +403 val_403 val_403 +404 val_404 val_404 +406 val_406 val_406 +407 val_407 val_407 +409 val_409 val_409 +41 val_41 val_41 +411 val_411 val_411 +413 val_413 val_413 +414 val_414 val_414 +417 val_417 val_417 +418 val_418 val_418 +419 val_419 val_419 +42 val_42 val_42 +421 val_421 val_421 +424 val_424 val_424 +427 val_427 val_427 +429 val_429 val_429 +43 val_43 val_43 +430 val_430 val_430 +431 val_431 val_431 +432 val_432 val_432 +435 val_435 val_435 +436 val_436 val_436 +437 val_437 val_437 +438 val_438 val_438 +439 val_439 val_439 +44 val_44 val_44 +443 val_443 val_443 +444 val_444 val_444 +446 val_446 val_446 +448 val_448 val_448 +449 val_449 val_449 +452 val_452 val_452 +453 val_453 val_453 +454 val_454 val_454 +455 val_455 val_455 +457 val_457 val_457 +458 val_458 val_458 +459 val_459 val_459 +460 val_460 val_460 +462 val_462 val_462 +463 val_463 val_463 +466 val_466 val_466 +467 val_467 val_467 +468 val_468 val_468 +469 val_469 val_469 +47 val_47 val_47 +470 val_470 val_470 +472 val_472 val_472 +475 val_475 val_475 +477 val_477 val_477 +478 val_478 val_478 +479 val_479 val_479 +480 val_480 val_480 +481 val_481 val_481 +482 val_482 val_482 +483 val_483 val_483 +484 val_484 val_484 +485 val_485 val_485 +487 val_487 val_487 +489 val_489 val_489 +490 val_490 val_490 +491 val_491 val_491 +492 val_492 val_492 +493 val_493 val_493 +494 val_494 val_494 +495 val_495 val_495 +496 val_496 val_496 +497 val_497 val_497 +498 val_498 val_498 +5 val_5 val_5 +51 val_51 val_51 +53 val_53 val_53 +54 val_54 val_54 +57 val_57 val_57 +58 val_58 val_58 +64 val_64 val_64 +65 val_65 val_65 +66 val_66 val_66 +67 val_67 val_67 +69 val_69 val_69 +70 val_70 val_70 +72 val_72 val_72 +74 val_74 val_74 +76 val_76 val_76 +77 val_77 val_77 +78 val_78 val_78 +8 val_8 val_8 +80 val_80 val_80 +82 val_82 val_82 +83 val_83 val_83 +84 val_84 val_84 +85 val_85 val_85 +86 val_86 val_86 +87 val_87 val_87 +9 val_9 val_9 +90 val_90 val_90 +92 val_92 val_92 +95 val_95 val_95 +96 val_96 val_96 +97 val_97 val_97 +98 val_98 val_98 +tst1 500 500 +PREHOOK: query: -- union2.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union2.q + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: -- union20.q + +-- SORT_BEFORE_DIFF +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +POSTHOOK: query: -- union20.q + +-- SORT_BEFORE_DIFF +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-5 is a root stage + Stage-6 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string) + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +tst1 500 tst1 500 +PREHOOK: query: -- union21.q + +-- SORT_BEFORE_DIFF +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +POSTHOOK: query: -- union21.q + +-- SORT_BEFORE_DIFF +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: reverse(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 47812 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 1000 Data size: 47812 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 85 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 501 Data size: 5397 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 501 Data size: 5397 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: astring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: lstring[0] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +NULL 1 +0 1 +001 1 +002 1 +004 1 +01 1 +013 1 +02 1 +021 1 +03 1 +032 1 +034 1 +051 1 +061 1 +062 1 +063 1 +064 1 +07 1 +071 1 +074 1 +08 1 +081 1 +082 1 +084 1 +09 1 +091 1 +094 1 +1 1 +10 1 +100 1 +102 1 +103 1 +104 1 +105 1 +11 1 +111 1 +113 1 +114 1 +116 1 +118 1 +119 1 +12 1 +120 1 +122 1 +123 1 +124 1 +125 1 +126 1 +128 1 +129 1 +131 1 +133 1 +134 1 +136 1 +137 1 +138 1 +14 1 +142 1 +143 1 +145 1 +146 1 +149 1 +15 1 +150 1 +152 1 +153 1 +155 1 +156 1 +157 1 +158 1 +160 1 +162 1 +163 1 +164 1 +165 1 +166 1 +167 1 +168 1 +169 1 +17 1 +170 1 +172 1 +174 1 +175 1 +176 1 +177 1 +178 1 +179 1 +18 1 +180 1 +181 1 +182 1 +183 1 +184 1 +186 1 +187 1 +189 1 +19 1 +190 1 +191 1 +192 1 +193 1 +194 1 +195 1 +196 1 +197 1 +199 1 +2 1 +20 1 +200 1 +201 1 +202 1 +203 1 +204 1 +205 1 +207 1 +208 1 +209 1 +21 1 +213 1 +214 1 +216 1 +217 1 +218 1 +219 1 +221 1 +222 1 +223 1 +224 1 +226 1 +228 1 +229 1 +230 1 +233 1 +234 1 +235 1 +237 1 +238 1 +239 1 +24 1 +241 1 +242 1 +243 1 +244 1 +247 1 +248 1 +249 1 +251 1 +252 1 +254 1 +255 1 +256 1 +257 1 +258 1 +26 1 +260 1 +261 1 +262 1 +263 1 +264 1 +265 1 +266 1 +27 1 +271 1 +272 1 +273 1 +274 1 +275 1 +277 1 +278 1 +28 1 +280 1 +281 1 +282 1 +283 1 +284 1 +285 1 +286 1 +287 1 +288 1 +289 1 +29 1 +291 1 +292 1 +293 1 +294 1 +296 1 +298 1 +30 1 +301 1 +302 1 +304 1 +305 1 +306 1 +307 1 +308 1 +309 1 +310 1 +311 1 +312 1 +314 1 +315 1 +316 1 +317 1 +318 1 +321 1 +322 1 +323 1 +325 1 +327 1 +33 1 +331 1 +332 1 +333 1 +335 1 +336 1 +338 1 +339 1 +34 1 +341 1 +342 1 +344 1 +345 1 +348 1 +35 1 +351 1 +353 1 +354 1 +356 1 +360 1 +361 1 +362 1 +364 1 +365 1 +366 1 +367 1 +368 1 +369 1 +37 1 +372 1 +373 1 +374 1 +375 1 +377 1 +378 1 +379 1 +38 1 +381 1 +382 1 +384 1 +386 1 +389 1 +391 1 +392 1 +393 1 +394 1 +395 1 +396 1 +397 1 +399 1 +4 1 +40 1 +400 1 +401 1 +402 1 +403 1 +404 1 +406 1 +407 1 +409 1 +41 1 +411 1 +412 1 +413 1 +414 1 +417 1 +418 1 +419 1 +42 1 +421 1 +422 1 +424 1 +427 1 +429 1 +43 1 +430 1 +431 1 +432 1 +435 1 +436 1 +437 1 +438 1 +439 1 +44 1 +442 1 +443 1 +444 1 +446 1 +448 1 +449 1 +45 1 +452 1 +453 1 +454 1 +455 1 +457 1 +458 1 +459 1 +46 1 +460 1 +461 1 +462 1 +463 1 +466 1 +467 1 +468 1 +469 1 +47 1 +470 1 +471 1 +472 1 +473 1 +475 1 +477 1 +478 1 +479 1 +48 1 +480 1 +481 1 +482 1 +483 1 +484 1 +485 1 +487 1 +489 1 +490 1 +491 1 +492 1 +493 1 +494 1 +495 1 +496 1 +497 1 +498 1 +5 1 +50 1 +501 1 +502 1 +503 1 +51 1 +513 1 +521 1 +523 1 +53 1 +532 1 +533 1 +534 1 +54 1 +541 1 +543 1 +551 1 +552 1 +554 1 +56 1 +561 1 +562 1 +563 1 +57 1 +571 1 +572 1 +573 1 +574 1 +58 1 +582 1 +584 1 +59 1 +591 1 +593 1 +594 1 +60 1 +603 1 +604 1 +611 1 +612 1 +613 1 +62 1 +621 1 +622 1 +631 1 +633 1 +634 1 +64 1 +641 1 +644 1 +65 1 +651 1 +652 1 +653 1 +66 1 +661 1 +662 1 +663 1 +664 1 +67 1 +671 1 +68 1 +681 1 +682 1 +683 1 +69 1 +691 1 +692 1 +693 1 +694 1 +70 1 +702 1 +703 1 +704 1 +71 1 +712 1 +713 1 +714 1 +72 1 +723 1 +724 1 +73 1 +731 1 +732 1 +734 1 +74 1 +742 1 +75 1 +751 1 +752 1 +754 1 +76 1 +761 1 +763 1 +764 1 +77 1 +771 1 +772 1 +773 1 +774 1 +78 1 +781 1 +782 1 +784 1 +79 1 +791 1 +793 1 +794 1 +8 1 +80 1 +802 1 +803 1 +81 1 +811 1 +812 1 +813 1 +814 1 +82 1 +821 1 +822 1 +83 1 +831 1 +832 1 +833 1 +834 1 +84 1 +842 1 +843 1 +844 1 +85 1 +851 1 +852 1 +854 1 +86 1 +861 1 +863 1 +864 1 +87 1 +871 1 +872 1 +873 1 +874 1 +882 1 +89 1 +892 1 +894 1 +9 1 +90 1 +902 1 +903 1 +904 1 +91 1 +911 1 +912 1 +914 1 +92 1 +921 1 +922 1 +924 1 +932 1 +933 1 +934 1 +941 1 +942 1 +944 1 +95 1 +954 1 +96 1 +961 1 +963 1 +964 1 +97 1 +971 1 +973 1 +974 1 +98 1 +981 1 +982 1 +983 1 +984 1 +991 1 +993 1 +record_0 1 +record_1 1 +record_2 1 +record_3 1 +record_4 1 +record_5 1 +record_6 1 +record_7 1 +record_8 1 +record_9 1 +PREHOOK: query: -- union22.q + +-- SORT_QUERY_RESULTS + +create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22 +POSTHOOK: query: -- union22.q + +-- SORT_QUERY_RESULTS + +create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22 +PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dst_union22_delta +POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dst_union22_delta +PREHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1') +select key, value, key , value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dst_union22@ds=1 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1') +select key, key, value, key, value, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dst_union22_delta@ds=1 +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin + +explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +PREHOOK: type: QUERY +POSTHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin + +explain extended +insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + dst_union22_delta + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + k1 + k1 + TOK_SELEXPR + TOK_TABLE_OR_COL + k2 + k2 + TOK_SELEXPR + TOK_TABLE_OR_COL + k3 + k3 + TOK_SELEXPR + TOK_TABLE_OR_COL + k4 + k4 + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + <= + TOK_TABLE_OR_COL + k0 + 50 + TOK_QUERY + TOK_FROM + TOK_LEFTOUTERJOIN + TOK_TABREF + TOK_TABNAME + dst_union22 + a + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + dst_union22_delta + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + > + TOK_TABLE_OR_COL + k0 + 50 + b + and + = + . + TOK_TABLE_OR_COL + a + k1 + . + TOK_TABLE_OR_COL + b + k1 + = + . + TOK_TABLE_OR_COL + a + ds + '1' + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + k1 + k1 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + k2 + k2 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + k3 + k3 + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + k4 + k4 + TOK_WHERE + > + . + TOK_TABLE_OR_COL + a + k1 + 20 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + subq + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dst_union22 + TOK_PARTSPEC + TOK_PARTVAL + ds + '2' + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-7 is a root stage , consists of Stage-8, Stage-4 + Stage-8 has a backup stage: Stage-4 + Stage-6 depends on stages: Stage-8 + Stage-2 depends on stages: Stage-4, Stage-6 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 + +STAGE PLANS: + Stage: Stage-7 + Conditional Operator + + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq-subquery2:_u1-subquery2:b:dst_union22_delta + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Alias -> Map Local Operator Tree: + subq-subquery2:_u1-subquery2:b:dst_union22_delta + TableScan + alias: dst_union22_delta + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((k0 > 50) and (k1 > 20)) (type: boolean) + Select Operator + expressions: k1 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col1, _col3, _col4 + HashTable Sink Operator + filter mappings: + 0 [1, 1] + filter predicates: + 0 {(ds = '1')} + 1 + keys: + 0 k1 (type: string) + 1 _col1 (type: string) + Position of Big Table: 0 + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: a + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (k1 > 20) (type: boolean) + Map Join Operator + condition map: + Left Outer Join0 to 1 + filter mappings: + 0 [1, 1] + filter predicates: + 0 {(ds = '1')} + 1 + keys: + 0 k1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col11, _col12 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col11 (type: string), _col12 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 11124 + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11624 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + name: default.dst_union22 +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22/ds=1 [subq-subquery2:_u1-subquery2:a] + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: dst_union22_delta + Statistics: Num rows: 500 Data size: 16936 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (k0 <= 50) (type: boolean) + Statistics: Num rows: 166 Data size: 5622 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 166 Data size: 5622 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + GatherStats: false + Union + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: ++++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 348 Data size: 9684 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10001 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22_delta/ds=1 [subq-subquery1:_u1-subquery1:dst_union22_delta] +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2/ + Statistics: Num rows: 174 Data size: 4842 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2 + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: dst_union22_delta + Statistics: Num rows: 500 Data size: 16936 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((k0 > 50) and (k1 > 20)) (type: boolean) + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k3 (type: string), k4 (type: string) + outputColumnNames: _col1, _col3, _col4 + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 1862 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col3 (type: string), _col4 (type: string) + auto parallelism: false + TableScan + alias: a + Statistics: Num rows: 500 Data size: 11124 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (k1 > 20) (type: boolean) + Statistics: Num rows: 166 Data size: 3693 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 166 Data size: 3693 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: k2 (type: string), ds (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 11124 + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11624 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k1,k2,k3,k4 + columns.comments + columns.types string:string:string:string +#### A masked pattern was here #### + name default.dst_union22 + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22 + name: default.dst_union22 +#### A masked pattern was here #### + Partition + base file name: ds=1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + numFiles 1 + numRows 500 + partition_columns ds + partition_columns.types string + rawDataSize 16936 + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 17436 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns k0,k1,k2,k3,k4,k5 + columns.comments + columns.types string:string:string:string:string:string +#### A masked pattern was here #### + name default.dst_union22_delta + partition_columns ds + partition_columns.types string + serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dst_union22_delta + name: default.dst_union22_delta + Truncated Path -> Alias: + /dst_union22/ds=1 [subq-subquery2:_u1-subquery2:a] + /dst_union22_delta/ds=1 [subq-subquery2:_u1-subquery2:b:dst_union22_delta] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + filter mappings: + 0 [1, 1] + filter predicates: + 0 {(VALUE._col3 = '1')} + 1 + keys: + 0 k1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col11, _col12 + Statistics: Num rows: 182 Data size: 4062 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col11 (type: string), _col12 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 4062 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + +PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=1 +PREHOOK: Input: default@dst_union22_delta +PREHOOK: Input: default@dst_union22_delta@ds=1 +PREHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2') +select * from +( +select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50 +UNION DISTINCT +select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4 +from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on +a.k1 = b.k1 and a.ds='1' +where a.k1 > 20 +) +subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=1 +POSTHOOK: Input: default@dst_union22_delta +POSTHOOK: Input: default@dst_union22_delta@ds=1 +POSTHOOK: Output: default@dst_union22@ds=2 +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ] +POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ] +PREHOOK: query: select * from dst_union22 where ds = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@dst_union22 +PREHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: select * from dst_union22 where ds = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dst_union22 +POSTHOOK: Input: default@dst_union22@ds=2 +#### A masked pattern was here #### +0 val_0 0 val_0 2 +10 val_10 10 val_10 2 +100 val_100 100 val_100 2 +103 val_103 103 val_103 2 +104 val_104 104 val_104 2 +105 val_105 105 val_105 2 +11 val_11 11 val_11 2 +111 val_111 111 val_111 2 +113 val_113 113 val_113 2 +114 val_114 114 val_114 2 +116 val_116 116 val_116 2 +118 val_118 118 val_118 2 +119 val_119 119 val_119 2 +12 val_12 12 val_12 2 +120 val_120 120 val_120 2 +125 val_125 125 val_125 2 +126 val_126 126 val_126 2 +128 val_128 128 val_128 2 +129 val_129 129 val_129 2 +131 val_131 131 val_131 2 +133 val_133 133 val_133 2 +134 val_134 134 val_134 2 +136 val_136 136 val_136 2 +137 val_137 137 val_137 2 +138 val_138 138 val_138 2 +143 val_143 143 val_143 2 +145 val_145 145 val_145 2 +146 val_146 146 val_146 2 +149 val_149 149 val_149 2 +15 val_15 15 val_15 2 +150 val_150 150 val_150 2 +152 val_152 152 val_152 2 +153 val_153 153 val_153 2 +155 val_155 155 val_155 2 +156 val_156 156 val_156 2 +157 val_157 157 val_157 2 +158 val_158 158 val_158 2 +160 val_160 160 val_160 2 +162 val_162 162 val_162 2 +163 val_163 163 val_163 2 +164 val_164 164 val_164 2 +165 val_165 165 val_165 2 +166 val_166 166 val_166 2 +167 val_167 167 val_167 2 +168 val_168 168 val_168 2 +169 val_169 169 val_169 2 +17 val_17 17 val_17 2 +170 val_170 170 val_170 2 +172 val_172 172 val_172 2 +174 val_174 174 val_174 2 +175 val_175 175 val_175 2 +176 val_176 176 val_176 2 +177 val_177 177 val_177 2 +178 val_178 178 val_178 2 +179 val_179 179 val_179 2 +18 val_18 18 val_18 2 +180 val_180 180 val_180 2 +181 val_181 181 val_181 2 +183 val_183 183 val_183 2 +186 val_186 186 val_186 2 +187 val_187 187 val_187 2 +189 val_189 189 val_189 2 +19 val_19 19 val_19 2 +190 val_190 190 val_190 2 +191 val_191 191 val_191 2 +192 val_192 192 val_192 2 +193 val_193 193 val_193 2 +194 val_194 194 val_194 2 +195 val_195 195 val_195 2 +196 val_196 196 val_196 2 +197 val_197 197 val_197 2 +199 val_199 199 val_199 2 +2 val_2 2 val_2 2 +20 val_20 20 val_20 2 +200 val_200 200 val_200 2 +201 val_201 201 val_201 2 +202 val_202 202 val_202 2 +203 val_203 203 val_203 2 +205 val_205 205 val_205 2 +207 val_207 207 val_207 2 +208 val_208 208 val_208 2 +209 val_209 209 val_209 2 +213 val_213 213 val_213 2 +214 val_214 214 val_214 2 +216 val_216 216 val_216 2 +217 val_217 217 val_217 2 +218 val_218 218 val_218 2 +219 val_219 219 val_219 2 +221 val_221 221 val_221 2 +222 val_222 222 val_222 2 +223 val_223 223 val_223 2 +224 val_224 224 val_224 2 +226 val_226 226 val_226 2 +228 val_228 228 val_228 2 +229 val_229 229 val_229 2 +230 val_230 230 val_230 2 +233 val_233 233 val_233 2 +235 val_235 235 val_235 2 +237 val_237 237 val_237 2 +238 val_238 238 val_238 2 +239 val_239 239 val_239 2 +24 val_24 NULL NULL 2 +24 val_24 24 val_24 2 +241 val_241 241 val_241 2 +242 val_242 242 val_242 2 +244 val_244 244 val_244 2 +247 val_247 247 val_247 2 +248 val_248 248 val_248 2 +249 val_249 249 val_249 2 +252 val_252 252 val_252 2 +255 val_255 255 val_255 2 +256 val_256 256 val_256 2 +257 val_257 257 val_257 2 +258 val_258 258 val_258 2 +26 val_26 NULL NULL 2 +26 val_26 26 val_26 2 +260 val_260 260 val_260 2 +262 val_262 262 val_262 2 +263 val_263 263 val_263 2 +265 val_265 265 val_265 2 +266 val_266 266 val_266 2 +27 val_27 NULL NULL 2 +27 val_27 27 val_27 2 +272 val_272 272 val_272 2 +273 val_273 273 val_273 2 +274 val_274 274 val_274 2 +275 val_275 275 val_275 2 +277 val_277 277 val_277 2 +278 val_278 278 val_278 2 +28 val_28 NULL NULL 2 +28 val_28 28 val_28 2 +280 val_280 280 val_280 2 +281 val_281 281 val_281 2 +282 val_282 282 val_282 2 +283 val_283 283 val_283 2 +284 val_284 284 val_284 2 +285 val_285 285 val_285 2 +286 val_286 286 val_286 2 +287 val_287 287 val_287 2 +288 val_288 288 val_288 2 +289 val_289 289 val_289 2 +291 val_291 291 val_291 2 +292 val_292 292 val_292 2 +296 val_296 296 val_296 2 +298 val_298 298 val_298 2 +30 val_30 NULL NULL 2 +30 val_30 30 val_30 2 +302 val_302 302 val_302 2 +305 val_305 305 val_305 2 +306 val_306 306 val_306 2 +307 val_307 307 val_307 2 +308 val_308 308 val_308 2 +309 val_309 309 val_309 2 +310 val_310 310 val_310 2 +311 val_311 311 val_311 2 +315 val_315 315 val_315 2 +316 val_316 316 val_316 2 +317 val_317 317 val_317 2 +318 val_318 318 val_318 2 +321 val_321 321 val_321 2 +322 val_322 322 val_322 2 +323 val_323 323 val_323 2 +325 val_325 325 val_325 2 +327 val_327 327 val_327 2 +33 val_33 NULL NULL 2 +33 val_33 33 val_33 2 +331 val_331 331 val_331 2 +332 val_332 332 val_332 2 +333 val_333 333 val_333 2 +335 val_335 335 val_335 2 +336 val_336 336 val_336 2 +338 val_338 338 val_338 2 +339 val_339 339 val_339 2 +34 val_34 NULL NULL 2 +34 val_34 34 val_34 2 +341 val_341 341 val_341 2 +342 val_342 342 val_342 2 +344 val_344 344 val_344 2 +345 val_345 345 val_345 2 +348 val_348 348 val_348 2 +35 val_35 NULL NULL 2 +35 val_35 35 val_35 2 +351 val_351 351 val_351 2 +353 val_353 353 val_353 2 +356 val_356 356 val_356 2 +360 val_360 360 val_360 2 +362 val_362 362 val_362 2 +364 val_364 364 val_364 2 +365 val_365 365 val_365 2 +366 val_366 366 val_366 2 +367 val_367 367 val_367 2 +368 val_368 368 val_368 2 +369 val_369 369 val_369 2 +37 val_37 NULL NULL 2 +37 val_37 37 val_37 2 +373 val_373 373 val_373 2 +374 val_374 374 val_374 2 +375 val_375 375 val_375 2 +377 val_377 377 val_377 2 +378 val_378 378 val_378 2 +379 val_379 379 val_379 2 +382 val_382 382 val_382 2 +384 val_384 384 val_384 2 +386 val_386 386 val_386 2 +389 val_389 389 val_389 2 +392 val_392 392 val_392 2 +393 val_393 393 val_393 2 +394 val_394 394 val_394 2 +395 val_395 395 val_395 2 +396 val_396 396 val_396 2 +397 val_397 397 val_397 2 +399 val_399 399 val_399 2 +4 val_4 4 val_4 2 +400 val_400 400 val_400 2 +401 val_401 401 val_401 2 +402 val_402 402 val_402 2 +403 val_403 403 val_403 2 +404 val_404 404 val_404 2 +406 val_406 406 val_406 2 +407 val_407 407 val_407 2 +409 val_409 409 val_409 2 +41 val_41 NULL NULL 2 +41 val_41 41 val_41 2 +411 val_411 411 val_411 2 +413 val_413 413 val_413 2 +414 val_414 414 val_414 2 +417 val_417 417 val_417 2 +418 val_418 418 val_418 2 +419 val_419 419 val_419 2 +42 val_42 NULL NULL 2 +42 val_42 42 val_42 2 +421 val_421 421 val_421 2 +424 val_424 424 val_424 2 +427 val_427 427 val_427 2 +429 val_429 429 val_429 2 +43 val_43 NULL NULL 2 +43 val_43 43 val_43 2 +430 val_430 430 val_430 2 +431 val_431 431 val_431 2 +432 val_432 432 val_432 2 +435 val_435 435 val_435 2 +436 val_436 436 val_436 2 +437 val_437 437 val_437 2 +438 val_438 438 val_438 2 +439 val_439 439 val_439 2 +44 val_44 NULL NULL 2 +44 val_44 44 val_44 2 +443 val_443 443 val_443 2 +444 val_444 444 val_444 2 +446 val_446 446 val_446 2 +448 val_448 448 val_448 2 +449 val_449 449 val_449 2 +452 val_452 452 val_452 2 +453 val_453 453 val_453 2 +454 val_454 454 val_454 2 +455 val_455 455 val_455 2 +457 val_457 457 val_457 2 +458 val_458 458 val_458 2 +459 val_459 459 val_459 2 +460 val_460 460 val_460 2 +462 val_462 462 val_462 2 +463 val_463 463 val_463 2 +466 val_466 466 val_466 2 +467 val_467 467 val_467 2 +468 val_468 468 val_468 2 +469 val_469 469 val_469 2 +47 val_47 NULL NULL 2 +47 val_47 47 val_47 2 +470 val_470 470 val_470 2 +472 val_472 472 val_472 2 +475 val_475 475 val_475 2 +477 val_477 477 val_477 2 +478 val_478 478 val_478 2 +479 val_479 479 val_479 2 +480 val_480 480 val_480 2 +481 val_481 481 val_481 2 +482 val_482 482 val_482 2 +483 val_483 483 val_483 2 +484 val_484 484 val_484 2 +485 val_485 485 val_485 2 +487 val_487 487 val_487 2 +489 val_489 489 val_489 2 +490 val_490 490 val_490 2 +491 val_491 491 val_491 2 +492 val_492 492 val_492 2 +493 val_493 493 val_493 2 +494 val_494 494 val_494 2 +495 val_495 495 val_495 2 +496 val_496 496 val_496 2 +497 val_497 497 val_497 2 +498 val_498 498 val_498 2 +5 val_5 5 val_5 2 +51 val_51 51 val_51 2 +53 val_53 53 val_53 2 +54 val_54 54 val_54 2 +57 val_57 57 val_57 2 +58 val_58 58 val_58 2 +64 val_64 64 val_64 2 +65 val_65 65 val_65 2 +66 val_66 66 val_66 2 +67 val_67 67 val_67 2 +69 val_69 69 val_69 2 +70 val_70 70 val_70 2 +72 val_72 72 val_72 2 +74 val_74 74 val_74 2 +76 val_76 76 val_76 2 +77 val_77 77 val_77 2 +78 val_78 78 val_78 2 +8 val_8 8 val_8 2 +80 val_80 80 val_80 2 +82 val_82 82 val_82 2 +83 val_83 83 val_83 2 +84 val_84 84 val_84 2 +85 val_85 85 val_85 2 +86 val_86 86 val_86 2 +87 val_87 87 val_87 2 +9 val_9 9 val_9 2 +90 val_90 90 val_90 2 +92 val_92 92 val_92 2 +95 val_95 95 val_95 2 +96 val_96 96 val_96 2 +97 val_97 97 val_97 2 +98 val_98 98 val_98 2 +PREHOOK: query: -- union23.q + +explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +PREHOOK: type: QUERY +POSTHOOK: query: -- union23.q + +explain +select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select s.key2, s.value2 +from ( + select transform(key, value) using 'cat' as (key2, value2) + from src + UNION DISTINCT + select key as key2, value as value2 from src) s +order by s.key2, s.value2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union24.q + +-- SORT_QUERY_RESULTS + +create table src2 as select key, count(1) as count from src group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src2 +POSTHOOK: query: -- union24.q + +-- SORT_QUERY_RESULTS + +create table src2 as select key, count(1) as count from src group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src2 +PREHOOK: query: create table src3 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src3 +POSTHOOK: query: create table src3 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src3 +PREHOOK: query: create table src4 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src4 +POSTHOOK: query: create table src4 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src4 +PREHOOK: query: create table src5 as select * from src2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src2 +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select * from src2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src4 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src5 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_FUNCTION + count + 1 + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_GROUPBY + TOK_TABLE_OR_COL + key + _u3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-6 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + alias: src3 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src2 [$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:src2] + /src3 [$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery2:$hdt$_0-subquery2:src3] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + alias: src4 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 + Truncated Path -> Alias: + /src4 [$hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0-subquery2:$hdt$_0-subquery2:src4] +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Union + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + GatherStats: false + Union + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 154 Data size: 738 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10004 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: -mr-10005 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 77 Data size: 369 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: src5 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [$hdt$_0-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_0:src5] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select key, count from src4 where key < 10 + UNION DISTINCT + select key, count(1) as count from src5 where key < 10 group by key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 1 +0 3 +2 1 +4 1 +5 1 +5 3 +8 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src4 + a + TOK_TABREF + TOK_TABNAME + src5 + b + = + . + TOK_TABLE_OR_COL + a + key + . + TOK_TABLE_OR_COL + b + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + key + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + b + count + count + TOK_WHERE + < + . + TOK_TABLE_OR_COL + a + key + 10 + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + alias: src3 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src2 [s-subquery1:_u2-subquery1-subquery1:_u1-subquery1:src2] + /src3 [s-subquery1:_u2-subquery1-subquery2:_u1-subquery2:src3] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + s-subquery2:_u2-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + s-subquery2:_u2-subquery2:a + TableScan + alias: a + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Union + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + alias: b + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col6 + Position of Big Table: 1 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col6 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 216 Data size: 1037 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [s-subquery2:_u2-subquery2:b] +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 108 Data size: 518 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, b.count as count from src4 a join src5 b on a.key=b.key where a.key < 10 +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_UNIONALL + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src3 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + count + TOK_WHERE + < + TOK_TABLE_OR_COL + key + 10 + _u1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src4 + a + TOK_TABREF + TOK_TABNAME + src5 + b + = + . + TOK_TABLE_OR_COL + a + key + . + TOK_TABLE_OR_COL + b + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + a + key + key + TOK_SELEXPR + TOK_FUNCTION + count + 1 + count + TOK_WHERE + < + . + TOK_TABLE_OR_COL + a + key + 10 + TOK_GROUPBY + . + TOK_TABLE_OR_COL + a + key + _u2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECTDI + TOK_SELEXPR + TOK_ALLCOLREF + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + s + count + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-8 is a root stage + Stage-5 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + alias: src3 + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), count (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 206 Data size: 988 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src2 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src2 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src2 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src2 + name: default.src2 +#### A masked pattern was here #### + Partition + base file name: src3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src3 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src3 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src3 + name: default.src3 + Truncated Path -> Alias: + /src2 [s-subquery1:_u2-subquery1-subquery1:_u1-subquery1:src2] + /src3 [s-subquery1:_u2-subquery1-subquery2:_u1-subquery2:src3] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Union + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + TableScan + GatherStats: false + Union + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 159 Data size: 763 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: -mr-10005 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 79 Data size: 379 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + s-subquery2:_u2-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + s-subquery2:_u2-subquery2:a + TableScan + alias: a + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Position of Big Table: 1 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src4 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src4 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src4 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src4 + name: default.src4 +#### A masked pattern was here #### + Partition + base file name: src5 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,count + columns.comments + columns.types string:bigint +#### A masked pattern was here #### + name default.src5 + numFiles 1 + numRows 309 + rawDataSize 1482 + serialization.ddl struct src5 { string key, i64 count} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1791 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src5 + name: default.src5 + Truncated Path -> Alias: + /src5 [s-subquery2:_u2-subquery2:b] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,bigint + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +PREHOOK: type: QUERY +PREHOOK: Input: default@src2 +PREHOOK: Input: default@src3 +PREHOOK: Input: default@src4 +PREHOOK: Input: default@src5 +#### A masked pattern was here #### +POSTHOOK: query: select s.key, s.count from ( + select key, count from src2 where key < 10 + UNION DISTINCT + select key, count from src3 where key < 10 + UNION DISTINCT + select a.key as key, count(1) as count from src4 a join src5 b on a.key=b.key where a.key < 10 group by a.key +)s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src2 +POSTHOOK: Input: default@src3 +POSTHOOK: Input: default@src4 +POSTHOOK: Input: default@src5 +#### A masked pattern was here #### +0 1 +0 3 +2 1 +4 1 +5 1 +5 3 +8 1 +9 1 +PREHOOK: query: -- union25.q + +create table tmp_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: -- union25.q + +create table tmp_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tmp_srcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tmp_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +PREHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: query: explain +create table tmp_unionall as +SELECT count(1) as counts, key, value +FROM +( + SELECT key, value FROM srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + + UNION DISTINCT + + SELECT key, key as value FROM ( + SELECT distinct key FROM ( + SELECT key, value FROM tmp_srcpart a WHERE a.ds='2008-04-08' and a.hr='11' + UNION DISTINCT + SELECT key, value FROM tmp_srcpart b WHERE b.ds='2008-04-08' and b.hr='11' + )t + ) master_table +) a GROUP BY key, value +POSTHOOK: type: CREATETABLE_AS_SELECT +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-2 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-2 + Stage-6 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-6 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 187 Data size: 1986 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-6 + Create Table Operator: + Create Table + columns: counts bigint, key string, value string + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_unionall + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: -- union26.q + +-- SORT_QUERY_RESULTS + +EXPLAIN +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +POSTHOOK: query: -- union26.q + +-- SORT_QUERY_RESULTS + +EXPLAIN +SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a-subquery1:_u1-subquery1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a-subquery1:_u1-subquery1:a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 318 Data size: 3378 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: SELECT +count(1) as counts, +key, +value +FROM +( + +SELECT +a.key, a.value +FROM srcpart a JOIN srcpart b +ON a.ds='2008-04-08' and a.hr='11' and b.ds='2008-04-08' and b.hr='12' +AND a.key = b.key + +UNION DISTINCT + +select key, value +FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +WHERE ds='2008-04-08' and hr='11' +) a +group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1 0 val_0 +1 10 val_10 +1 100 val_100 +1 103 val_103 +1 104 val_104 +1 105 val_105 +1 11 val_11 +1 111 val_111 +1 113 val_113 +1 114 val_114 +1 116 val_116 +1 118 val_118 +1 119 val_119 +1 12 val_12 +1 120 val_120 +1 125 val_125 +1 126 val_126 +1 128 val_128 +1 129 val_129 +1 131 val_131 +1 133 val_133 +1 134 val_134 +1 136 val_136 +1 137 val_137 +1 138 val_138 +1 143 val_143 +1 145 val_145 +1 146 val_146 +1 149 val_149 +1 15 val_15 +1 150 val_150 +1 152 val_152 +1 153 val_153 +1 155 val_155 +1 156 val_156 +1 157 val_157 +1 158 val_158 +1 160 val_160 +1 162 val_162 +1 163 val_163 +1 164 val_164 +1 165 val_165 +1 166 val_166 +1 167 val_167 +1 168 val_168 +1 169 val_169 +1 17 val_17 +1 170 val_170 +1 172 val_172 +1 174 val_174 +1 175 val_175 +1 176 val_176 +1 177 val_177 +1 178 val_178 +1 179 val_179 +1 18 val_18 +1 180 val_180 +1 181 val_181 +1 183 val_183 +1 186 val_186 +1 187 val_187 +1 189 val_189 +1 19 val_19 +1 190 val_190 +1 191 val_191 +1 192 val_192 +1 193 val_193 +1 194 val_194 +1 195 val_195 +1 196 val_196 +1 197 val_197 +1 199 val_199 +1 2 val_2 +1 20 val_20 +1 200 val_200 +1 201 val_201 +1 202 val_202 +1 203 val_203 +1 205 val_205 +1 207 val_207 +1 208 val_208 +1 209 val_209 +1 213 val_213 +1 214 val_214 +1 216 val_216 +1 217 val_217 +1 218 val_218 +1 219 val_219 +1 221 val_221 +1 222 val_222 +1 223 val_223 +1 224 val_224 +1 226 val_226 +1 228 val_228 +1 229 val_229 +1 230 val_230 +1 233 val_233 +1 235 val_235 +1 237 val_237 +1 238 val_238 +1 239 val_239 +1 24 val_24 +1 241 val_241 +1 242 val_242 +1 244 val_244 +1 247 val_247 +1 248 val_248 +1 249 val_249 +1 252 val_252 +1 255 val_255 +1 256 val_256 +1 257 val_257 +1 258 val_258 +1 26 val_26 +1 260 val_260 +1 262 val_262 +1 263 val_263 +1 265 val_265 +1 266 val_266 +1 27 val_27 +1 272 val_272 +1 273 val_273 +1 274 val_274 +1 275 val_275 +1 277 val_277 +1 278 val_278 +1 28 val_28 +1 280 val_280 +1 281 val_281 +1 282 val_282 +1 283 val_283 +1 284 val_284 +1 285 val_285 +1 286 val_286 +1 287 val_287 +1 288 val_288 +1 289 val_289 +1 291 val_291 +1 292 val_292 +1 296 val_296 +1 298 val_298 +1 30 val_30 +1 302 val_302 +1 305 val_305 +1 306 val_306 +1 307 val_307 +1 308 val_308 +1 309 val_309 +1 310 val_310 +1 311 val_311 +1 315 val_315 +1 316 val_316 +1 317 val_317 +1 318 val_318 +1 321 val_321 +1 322 val_322 +1 323 val_323 +1 325 val_325 +1 327 val_327 +1 33 val_33 +1 331 val_331 +1 332 val_332 +1 333 val_333 +1 335 val_335 +1 336 val_336 +1 338 val_338 +1 339 val_339 +1 34 val_34 +1 341 val_341 +1 342 val_342 +1 344 val_344 +1 345 val_345 +1 348 val_348 +1 35 val_35 +1 351 val_351 +1 353 val_353 +1 356 val_356 +1 360 val_360 +1 362 val_362 +1 364 val_364 +1 365 val_365 +1 366 val_366 +1 367 val_367 +1 368 val_368 +1 369 val_369 +1 37 val_37 +1 373 val_373 +1 374 val_374 +1 375 val_375 +1 377 val_377 +1 378 val_378 +1 379 val_379 +1 382 val_382 +1 384 val_384 +1 386 val_386 +1 389 val_389 +1 392 val_392 +1 393 val_393 +1 394 val_394 +1 395 val_395 +1 396 val_396 +1 397 val_397 +1 399 val_399 +1 4 val_4 +1 400 val_400 +1 401 val_401 +1 402 val_402 +1 403 val_403 +1 404 val_404 +1 406 val_406 +1 407 val_407 +1 409 val_409 +1 41 val_41 +1 411 val_411 +1 413 val_413 +1 414 val_414 +1 417 val_417 +1 418 val_418 +1 419 val_419 +1 42 val_42 +1 421 val_421 +1 424 val_424 +1 427 val_427 +1 429 val_429 +1 43 val_43 +1 430 val_430 +1 431 val_431 +1 432 val_432 +1 435 val_435 +1 436 val_436 +1 437 val_437 +1 438 val_438 +1 439 val_439 +1 44 val_44 +1 443 val_443 +1 444 val_444 +1 446 val_446 +1 448 val_448 +1 449 val_449 +1 452 val_452 +1 453 val_453 +1 454 val_454 +1 455 val_455 +1 457 val_457 +1 458 val_458 +1 459 val_459 +1 460 val_460 +1 462 val_462 +1 463 val_463 +1 466 val_466 +1 467 val_467 +1 468 val_468 +1 469 val_469 +1 47 val_47 +1 470 val_470 +1 472 val_472 +1 475 val_475 +1 477 val_477 +1 478 val_478 +1 479 val_479 +1 480 val_480 +1 481 val_481 +1 482 val_482 +1 483 val_483 +1 484 val_484 +1 485 val_485 +1 487 val_487 +1 489 val_489 +1 490 val_490 +1 491 val_491 +1 492 val_492 +1 493 val_493 +1 494 val_494 +1 495 val_495 +1 496 val_496 +1 497 val_497 +1 498 val_498 +1 5 val_5 +1 51 val_51 +1 53 val_53 +1 54 val_54 +1 57 val_57 +1 58 val_58 +1 64 val_64 +1 65 val_65 +1 66 val_66 +1 67 val_67 +1 69 val_69 +1 70 val_70 +1 72 val_72 +1 74 val_74 +1 76 val_76 +1 77 val_77 +1 78 val_78 +1 8 val_8 +1 80 val_80 +1 82 val_82 +1 83 val_83 +1 84 val_84 +1 85 val_85 +1 86 val_86 +1 87 val_87 +1 9 val_9 +1 90 val_90 +1 92 val_92 +1 95 val_95 +1 96 val_96 +1 97 val_97 +1 98 val_98 +PREHOOK: query: -- union27.q + +-- SORT_BEFORE_DIFF +create table jackson_sev_same as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_same +POSTHOOK: query: -- union27.q + +-- SORT_BEFORE_DIFF +create table jackson_sev_same as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_same +PREHOOK: query: create table dim_pho as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dim_pho +POSTHOOK: query: create table dim_pho as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dim_pho +PREHOOK: query: create table jackson_sev_add as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@jackson_sev_add +POSTHOOK: query: create table jackson_sev_add as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@jackson_sev_add +PREHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +POSTHOOK: query: explain select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 + Stage-5 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: dim_pho + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '97' (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: jackson_sev_add + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '97' (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 '97' (type: string) + 1 '97' (type: string) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '97' (type: string) + 1 '97' (type: string) + outputColumnNames: _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '97' (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +PREHOOK: type: QUERY +PREHOOK: Input: default@dim_pho +PREHOOK: Input: default@jackson_sev_add +PREHOOK: Input: default@jackson_sev_same +#### A masked pattern was here #### +POSTHOOK: query: select b.* from jackson_sev_same a join (select * from dim_pho UNION DISTINCT select * from jackson_sev_add)b on a.key=b.key and b.key=97 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dim_pho +POSTHOOK: Input: default@jackson_sev_add +POSTHOOK: Input: default@jackson_sev_same +#### A masked pattern was here #### +97 val_97 +97 val_97 +PREHOOK: query: -- union28.q + +create table union_subq_union(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: -- union28.q + +create table union_subq_union(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union +PREHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-5 depends on stages: Stage-4, Stage-6 + Stage-2 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-6 is a root stage + +STAGE PLANS: + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union +POSTHOOK: query: insert overwrite table union_subq_union +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union +POSTHOOK: Lineage: union_subq_union.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union29.q + +create table union_subq_union29(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: -- union29.q + +create table union_subq_union29(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union29 +PREHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-2 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union29 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union29 +POSTHOOK: query: insert overwrite table union_subq_union29 +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value from src + UNION DISTINCT + select key, value from src + ) subq +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union29 +POSTHOOK: Lineage: union_subq_union29.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union29.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union29 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union29 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union29 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union29 +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union3.q + +-- SORT_BEFORE_DIFF + +explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +PREHOOK: type: QUERY +POSTHOOK: query: -- union3.q + +-- SORT_BEFORE_DIFF + +explain +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-3 depends on stages: Stage-2, Stage-6 + Stage-4 depends on stages: Stage-3, Stage-7 + Stage-5 is a root stage + Stage-6 is a root stage + Stage-7 is a root stage + Stage-0 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 3 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Operator Tree: + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 4 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE union_out (id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_out +POSTHOOK: query: CREATE TABLE union_out (id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_out +PREHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_out +POSTHOOK: query: insert overwrite table union_out +SELECT * +FROM ( + SELECT 1 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 2 AS id + FROM (SELECT * FROM src LIMIT 1) s1 + UNION DISTINCT + SELECT 3 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + UNION DISTINCT + SELECT 4 AS id + FROM (SELECT * FROM src LIMIT 1) s2 + CLUSTER BY id +) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_out +POSTHOOK: Lineage: union_out.id EXPRESSION [] +PREHOOK: query: select * from union_out +PREHOOK: type: QUERY +PREHOOK: Input: default@union_out +#### A masked pattern was here #### +POSTHOOK: query: select * from union_out +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_out +#### A masked pattern was here #### +1 +2 +3 +4 +PREHOOK: query: -- union30.q + +create table union_subq_union30(key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: -- union30.q + +create table union_subq_union30(key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_subq_union30 +PREHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-6 depends on stages: Stage-5, Stage-7 + Stage-2 depends on stages: Stage-6 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-7 is a root stage + +STAGE PLANS: + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 750 Data size: 7968 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 375 Data size: 3984 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 875 Data size: 9296 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 437 Data size: 4642 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union30 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union30 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@union_subq_union30 +POSTHOOK: query: insert overwrite table union_subq_union30 +select * from ( + +select * from ( + select key, value from src + UNION DISTINCT + select key, value from + ( + select key, value, count(1) from src group by key, value + UNION DISTINCT + select key, value, count(1) from src group by key, value + ) subq +) a + +UNION DISTINCT + +select key, value from src +) aa +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@union_subq_union30 +POSTHOOK: Lineage: union_subq_union30.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: union_subq_union30.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from union_subq_union30 order by key, value limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@union_subq_union30 +#### A masked pattern was here #### +POSTHOOK: query: select * from union_subq_union30 order by key, value limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_subq_union30 +#### A masked pattern was here #### +0 val_0 +2 val_2 +4 val_4 +5 val_5 +8 val_8 +9 val_9 +10 val_10 +11 val_11 +12 val_12 +15 val_15 +17 val_17 +18 val_18 +19 val_19 +20 val_20 +24 val_24 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +PREHOOK: query: -- union31.q + +-- SORT_QUERY_RESULTS + +drop table t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- union31.q + +-- SORT_QUERY_RESULTS + +drop table t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table t2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t2 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t3(key string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3 +POSTHOOK: query: create table t3(key string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3 +PREHOOK: query: create table t4(value string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t4 +POSTHOOK: query: create table t4(value string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4 +PREHOOK: query: explain +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t3 + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t4 + + Stage: Stage-5 + Stats-Aggr Operator + +PREHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t3 +PREHOOK: Output: default@t4 +POSTHOOK: query: from +(select * from t1 + UNION DISTINCT + select * from t2 +) x +insert overwrite table t3 + select key, count(1) group by key +insert overwrite table t4 + select value, count(1) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t3 +POSTHOOK: Output: default@t4 +POSTHOOK: Lineage: t3.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t3.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t4.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select * from t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t4 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +POSTHOOK: query: select * from t4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4 +#### A masked pattern was here #### +val_0 1 +val_2 1 +val_4 1 +val_5 1 +val_8 1 +val_9 1 +PREHOOK: query: create table t5(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t5 +POSTHOOK: query: create table t5(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t5 +PREHOOK: query: create table t6(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t6 +POSTHOOK: query: create table t6(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t6 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2, Stage-7 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-1 + Stage-7 is a root stage + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Forward + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t5 + + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t6 + + Stage: Stage-6 + Stats-Aggr Operator + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t5 +PREHOOK: Output: default@t6 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, count(1) as cnt from t2 group by key +) x +insert overwrite table t5 + select c1, sum(cnt) group by c1 +insert overwrite table t6 + select c1, sum(cnt) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t5 +POSTHOOK: Output: default@t6 +POSTHOOK: Lineage: t5.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t5.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t6.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t6.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +PREHOOK: query: select * from t5 +PREHOOK: type: QUERY +PREHOOK: Input: default@t5 +#### A masked pattern was here #### +POSTHOOK: query: select * from t5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t5 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: select * from t6 +PREHOOK: type: QUERY +PREHOOK: Input: default@t6 +#### A masked pattern was here #### +POSTHOOK: query: select * from t6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t6 +#### A masked pattern was here #### +0 3 +2 1 +4 1 +5 3 +8 1 +9 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: drop table t2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t2 +POSTHOOK: query: drop table t2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t1 as select * from src where key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 as select * from src where key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 as select key, count(1) as cnt from src where key < 10 group by key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: create table t7(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t7 +POSTHOOK: query: create table t7(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t7 +PREHOOK: query: create table t8(c1 string, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8(c1 string, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: t2 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), cnt (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 11 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Forward + Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7 + Group By Operator + aggregations: count(1) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t7 + + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t8 + + Stage: Stage-6 + Stats-Aggr Operator + +PREHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t7 +PREHOOK: Output: default@t8 +POSTHOOK: query: from +( + select key as c1, count(1) as cnt from t1 group by key + UNION DISTINCT + select key as c1, cnt from t2 +) x +insert overwrite table t7 + select c1, count(1) group by c1 +insert overwrite table t8 + select c1, count(1) group by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t7 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t7.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t7.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +POSTHOOK: Lineage: t8.c1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t8.cnt EXPRESSION [(t1)t1.null, (t2)t2.null, ] +PREHOOK: query: select * from t7 +PREHOOK: type: QUERY +PREHOOK: Input: default@t7 +#### A masked pattern was here #### +POSTHOOK: query: select * from t7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t7 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: select * from t8 +PREHOOK: type: QUERY +PREHOOK: Input: default@t8 +#### A masked pattern was here #### +POSTHOOK: query: select * from t8 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8 +#### A masked pattern was here #### +0 1 +2 1 +4 1 +5 1 +8 1 +9 1 +PREHOOK: query: -- union32.q + +-- SORT_QUERY_RESULTS + +-- This tests various union queries which have columns on one side of the query +-- being of double type and those on the other side another + +drop table if exists t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: -- union32.q + +-- SORT_QUERY_RESULTS + +-- This tests various union queries which have columns on one side of the query +-- being of double type and those on the other side another + +drop table if exists t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: drop table if exists t2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t2 +POSTHOOK: query: drop table if exists t2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t2 +PREHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: -- Test simple union with double +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +ORDER BY key +PREHOOK: type: QUERY +POSTHOOK: query: -- Test simple union with double +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +ORDER BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(UDFToLong(key)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 20 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t1 +UNION DISTINCT +SELECT CAST(key AS BIGINT) AS key FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the left +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the left +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a-subquery1:_u1-subquery1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a-subquery1:_u1-subquery1:a + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the right +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the right +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a-subquery2:_u1-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a-subquery2:_u1-subquery2:a + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 +2.0 +4.0 +5.0 +8.0 +9.0 +PREHOOK: query: -- Test union with join on the left selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the left selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a-subquery1:_u1-subquery1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a-subquery1:_u1-subquery1:a + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToString(UDFToDouble(_col5)) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key +UNION DISTINCT +SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0 +0.0 0.0 +2.0 2 +2.0 2.0 +4.0 4 +4.0 4.0 +5.0 5 +5.0 5.0 +8.0 8 +8.0 8.0 +9.0 9 +9.0 9.0 +PREHOOK: query: -- Test union with join on the right selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +POSTHOOK: query: -- Test union with join on the right selecting multiple columns +EXPLAIN +SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a-subquery2:_u1-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a-subquery2:_u1-subquery2:a + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: double) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), UDFToDouble(_col5) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: double), _col1 (type: double) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Statistics: Num rows: 15 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 50 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM +(SELECT CAST(key AS DOUBLE) AS key, CAST(key AS STRING) AS value FROM t2 +UNION DISTINCT +SELECT CAST(a.key AS BIGINT) AS key, CAST(b.key AS DOUBLE) AS value FROM t1 a JOIN t2 b ON a.key = b.key) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +0.0 0.0 +2.0 2.0 +4.0 4.0 +5.0 5.0 +8.0 8.0 +9.0 9.0 +PREHOOK: query: -- union33.q + +-- SORT_BEFORE_DIFF +-- This tests that a UNION DISTINCT with a map only subquery on one side and a +-- subquery involving two map reduce jobs on the other runs correctly. + +drop table if exists test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- union33.q + +-- SORT_BEFORE_DIFF +-- This tests that a UNION DISTINCT with a map only subquery on one side and a +-- subquery involving two map reduce jobs on the other runs correctly. + +drop table if exists test_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_src +POSTHOOK: query: CREATE TABLE test_src (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_src +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-2 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '0' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, value FROM src + WHERE key = 0 +UNION DISTINCT + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (src)src.null, ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), UDFToString(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '0' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_src + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@test_src +POSTHOOK: query: INSERT OVERWRITE TABLE test_src +SELECT key, value FROM ( + SELECT key, COUNT(*) AS value FROM src + GROUP BY key +UNION DISTINCT + SELECT key, value FROM src + WHERE key = 0 +)a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@test_src +POSTHOOK: Lineage: test_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_src.value EXPRESSION [(src)src.null, (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT COUNT(*) FROM test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@test_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_src +#### A masked pattern was here #### +310 +PREHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_1 +POSTHOOK: query: -- union34.q + +create table src10_1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_1 +PREHOOK: query: create table src10_2 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_2 +POSTHOOK: query: create table src10_2 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_2 +PREHOOK: query: create table src10_3 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_3 +POSTHOOK: query: create table src10_3 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_3 +PREHOOK: query: create table src10_4 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10_4 +POSTHOOK: query: create table src10_4 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10_4 +PREHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src10_1 +PREHOOK: Output: default@src10_2 +PREHOOK: Output: default@src10_3 +PREHOOK: Output: default@src10_4 +POSTHOOK: query: from (select * from src tablesample (10 rows)) a +insert overwrite table src10_1 select * +insert overwrite table src10_2 select * +insert overwrite table src10_3 select * +insert overwrite table src10_4 select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src10_1 +POSTHOOK: Output: default@src10_2 +POSTHOOK: Output: default@src10_3 +POSTHOOK: Output: default@src10_4 +POSTHOOK: Lineage: src10_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src10_4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, +-- we can use a single MR job to evaluate this entire query. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +POSTHOOK: query: -- When we convert the Join of sub1 and sub0 into a MapJoin, +-- we can use a single MR job to evaluate this entire query. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-8 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-8 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: src10_3 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src10_4 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + alias1-subquery1:_u2-subquery1:sub1:src10_1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + alias1-subquery1:_u2-subquery1:sub1:src10_1 + TableScan + alias: src10_1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src10_2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 +#### A masked pattern was here #### +165 val_165 +238 val_238 +255 val_255 +27 val_27 +278 val_278 +311 val_311 +409 val_409 +484 val_484 +86 val_86 +98 val_98 +PREHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, +-- we need to use two MR jobs to evaluate this query. +-- The first job is for the Join of sub1 and sub2. The second job +-- is for the UNION DISTINCT and ORDER BY. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +POSTHOOK: query: -- When we do not convert the Join of sub1 and sub0 into a MapJoin, +-- we need to use two MR jobs to evaluate this query. +-- The first job is for the Join of sub1 and sub2. The second job +-- is for the UNION DISTINCT and ORDER BY. +explain +SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-3 depends on stages: Stage-2 + Stage-5 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src10_1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + alias: src10_2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + TableScan + Union + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 15 Data size: 161 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 7 Data size: 75 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: src10_3 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src10_4 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src10_1 +PREHOOK: Input: default@src10_2 +PREHOOK: Input: default@src10_3 +PREHOOK: Input: default@src10_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM ( + SELECT sub1.key,sub1.value FROM (SELECT * FROM src10_1) sub1 JOIN (SELECT * FROM src10_2) sub0 ON (sub0.key = sub1.key) + UNION DISTINCT + SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 +) alias1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src10_1 +POSTHOOK: Input: default@src10_2 +POSTHOOK: Input: default@src10_3 +POSTHOOK: Input: default@src10_4 +#### A masked pattern was here #### +165 val_165 +238 val_238 +255 val_255 +27 val_27 +278 val_278 +311 val_311 +409 val_409 +484 val_484 +86 val_86 +98 val_98 +PREHOOK: query: -- union4.q + + + +-- union case: both subqueries are map-reduce jobs on same input, followed by filesink + +drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union4.q + + + +-- union case: both subqueries are map-reduce jobs on same input, followed by filesink + +drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: create table tmptable(key string, value int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [] +POSTHOOK: Lineage: tmptable.value EXPRESSION [] +PREHOOK: query: select * from tmptable x sort by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### +tst1 500 +tst2 500 +PREHOOK: query: -- union5.q + + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union5.q + + +-- SORT_BEFORE_DIFF +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Union + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +PREHOOK: query: -- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +tst1 500 +PREHOOK: query: -- union7.q + + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: -- union7.q + + + +-- SORT_BEFORE_DIFF +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: -- union8.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union8.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +119 val_119 +12 val_12 +120 val_120 +125 val_125 +126 val_126 +128 val_128 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +136 val_136 +137 val_137 +138 val_138 +143 val_143 +145 val_145 +146 val_146 +149 val_149 +15 val_15 +150 val_150 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +165 val_165 +166 val_166 +167 val_167 +168 val_168 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +174 val_174 +175 val_175 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +192 val_192 +193 val_193 +194 val_194 +195 val_195 +196 val_196 +197 val_197 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +205 val_205 +207 val_207 +208 val_208 +209 val_209 +213 val_213 +214 val_214 +216 val_216 +217 val_217 +218 val_218 +219 val_219 +221 val_221 +222 val_222 +223 val_223 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +230 val_230 +233 val_233 +235 val_235 +237 val_237 +238 val_238 +239 val_239 +24 val_24 +241 val_241 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +273 val_273 +274 val_274 +275 val_275 +277 val_277 +278 val_278 +28 val_28 +280 val_280 +281 val_281 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +30 val_30 +302 val_302 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +309 val_309 +310 val_310 +311 val_311 +315 val_315 +316 val_316 +317 val_317 +318 val_318 +321 val_321 +322 val_322 +323 val_323 +325 val_325 +327 val_327 +33 val_33 +331 val_331 +332 val_332 +333 val_333 +335 val_335 +336 val_336 +338 val_338 +339 val_339 +34 val_34 +341 val_341 +342 val_342 +344 val_344 +345 val_345 +348 val_348 +35 val_35 +351 val_351 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 +367 val_367 +368 val_368 +369 val_369 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +396 val_396 +397 val_397 +399 val_399 +4 val_4 +400 val_400 +401 val_401 +402 val_402 +403 val_403 +404 val_404 +406 val_406 +407 val_407 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +414 val_414 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +421 val_421 +424 val_424 +427 val_427 +429 val_429 +43 val_43 +430 val_430 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +439 val_439 +44 val_44 +443 val_443 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 +454 val_454 +455 val_455 +457 val_457 +458 val_458 +459 val_459 +460 val_460 +462 val_462 +463 val_463 +466 val_466 +467 val_467 +468 val_468 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +479 val_479 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +5 val_5 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +69 val_69 +70 val_70 +72 val_72 +74 val_74 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +92 val_92 +95 val_95 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: -- union9.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: -- union9.q + +-- SORT_BEFORE_DIFF +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 diff --git a/ql/src/test/results/clientpositive/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/unionDistinct_2.q.out new file mode 100644 index 0000000..e7b8df9 --- /dev/null +++ b/ql/src/test/results/clientpositive/unionDistinct_2.q.out @@ -0,0 +1,410 @@ +PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u1 +POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u1 +PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u2 +POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u2 +PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@u3 +POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u3 +PREHOOK: query: select * from u1 +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +#### A masked pattern was here #### +POSTHOOK: query: select * from u1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: select * from u2 +PREHOOK: type: QUERY +PREHOOK: Input: default@u2 +#### A masked pattern was here #### +POSTHOOK: query: select * from u2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u2 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +PREHOOK: query: select * from u3 +PREHOOK: type: QUERY +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select * from u3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +0 val_0 +0 val_0 +0 val_0 +PREHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union all +select key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union all +select key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union distinct +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union distinct +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +98 val_98 +98 val_98 +97 val_97 +97 val_97 +96 val_96 +PREHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union +select key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union all +select key, value from u2 +union +select key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from +( +select key, value from u1 +union +select key, value from u2 +union +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: drop view if exists v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select distinct * from +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: describe extended v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: describe extended v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +key string +value string + +#### A masked pattern was here #### +( +select distinct * from u1 +union +select key, value from u2 +union all +select key as key, value from u3 +) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from +( +select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` +union +select `u2`.`key`, `u2`.`value` from `default`.`u2` +union all +select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` +) `tab`, tableType:VIRTUAL_VIEW) +PREHOOK: query: select * from v +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +POSTHOOK: Input: default@v +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 +PREHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union +select key as key, value from u3 +) tab +PREHOOK: type: QUERY +PREHOOK: Input: default@u1 +PREHOOK: Input: default@u2 +PREHOOK: Input: default@u3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct * from +( +select key, value from u1 +union all +select key, value from u2 +union +select key as key, value from u3 +) tab +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u1 +POSTHOOK: Input: default@u2 +POSTHOOK: Input: default@u3 +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +96 val_96 +97 val_97 +98 val_98 diff --git a/ql/src/test/results/clientpositive/union_null.q.out b/ql/src/test/results/clientpositive/union_null.q.out index e2897ff..32cdf65 100644 --- a/ql/src/test/results/clientpositive/union_null.q.out +++ b/ql/src/test/results/clientpositive/union_null.q.out @@ -1,14 +1,14 @@ PREHOOK: query: -- SORT_BEFORE_DIFF -- HIVE-2901 -select x from (select value as x from src order by x limit 5 union all select NULL as x from src limit 5) a +select x from (select * from (select value as x from src order by x limit 5)a union all select * from (select NULL as x from src limit 5)b )a PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### POSTHOOK: query: -- SORT_BEFORE_DIFF -- HIVE-2901 -select x from (select value as x from src order by x limit 5 union all select NULL as x from src limit 5) a +select x from (select * from (select value as x from src order by x limit 5)a union all select * from (select NULL as x from src limit 5)b )a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -23,12 +23,12 @@ NULL NULL NULL PREHOOK: query: -- HIVE-4837 -select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a +select * from (select * from (select null as N from src1 group by key)a UNION ALL select * from (select null as N from src1 group by key)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### POSTHOOK: query: -- HIVE-4837 -select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a +select * from (select * from (select null as N from src1 group by key)a UNION ALL select * from (select null as N from src1 group by key)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_ppr.q.out b/ql/src/test/results/clientpositive/union_ppr.q.out index 0610337..ef875f3 100644 --- a/ql/src/test/results/clientpositive/union_ppr.q.out +++ b/ql/src/test/results/clientpositive/union_ppr.q.out @@ -21,7 +21,7 @@ ABSTRACT SYNTAX TREE: TOK_QUERY TOK_FROM TOK_SUBQUERY - TOK_UNION + TOK_UNIONALL TOK_QUERY TOK_FROM TOK_TABREF diff --git a/ql/src/test/results/clientpositive/union_remove_25.q.out b/ql/src/test/results/clientpositive/union_remove_25.q.out index 4eae6d2..8c93ead 100644 --- a/ql/src/test/results/clientpositive/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/union_remove_25.q.out @@ -257,18 +257,18 @@ PREHOOK: query: explain insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -368,9 +368,9 @@ STAGE PLANS: PREHOOK: query: insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -380,9 +380,9 @@ PREHOOK: Output: default@outputtbl2 POSTHOOK: query: insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)a UNION ALL - SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500 + select * from (SELECT key, value, ds from srcpart where ds='2008-04-08' limit 500)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart @@ -442,17 +442,17 @@ Storage Desc Params: PREHOOK: query: explain insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -553,9 +553,9 @@ STAGE PLANS: PREHOOK: query: insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -565,9 +565,9 @@ PREHOOK: Output: default@outputtbl3 POSTHOOK: query: insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)a UNION ALL - SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000 + select * from (SELECT key, value, ds, hr from srcpart where ds='2008-04-08' limit 1000)b ) a POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/union_top_level.q.out b/ql/src/test/results/clientpositive/union_top_level.q.out index 72e684c..baab030 100644 --- a/ql/src/test/results/clientpositive/union_top_level.q.out +++ b/ql/src/test/results/clientpositive/union_top_level.q.out @@ -2,21 +2,21 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- top level explain -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY POSTHOOK: query: -- SORT_QUERY_RESULTS -- top level explain -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -183,19 +183,19 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select key, 0 as value from src where key % 3 == 0 limit 3 +PREHOOK: query: select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select key, 0 as value from src where key % 3 == 0 limit 3 +POSTHOOK: query: select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -209,14 +209,14 @@ POSTHOOK: Input: default@src 484 1 86 2 PREHOOK: query: explain -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b PREHOOK: type: QUERY POSTHOOK: query: explain -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -425,15 +425,15 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +PREHOOK: query: select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +POSTHOOK: query: select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)a union all -select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10 +select * from (select s1.key as k, s2.value as v from src s1 join src s2 on (s1.key = s2.key) limit 10)b POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -460,20 +460,20 @@ POSTHOOK: Input: default@src PREHOOK: query: -- ctas explain create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: -- ctas explain create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: Stage-1 is a root stage @@ -702,21 +702,21 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe PREHOOK: query: create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@union_top POSTHOOK: query: create table union_top as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -747,20 +747,20 @@ POSTHOOK: Output: default@union_top PREHOOK: query: -- insert into explain insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY POSTHOOK: query: -- insert into explain insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -971,20 +971,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe PREHOOK: query: insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_top POSTHOOK: query: insert into table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_top @@ -1009,19 +1009,19 @@ POSTHOOK: Input: default@union_top 86 2 PREHOOK: query: explain insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1232,20 +1232,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe PREHOOK: query: insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_top POSTHOOK: query: insert overwrite table union_top -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@union_top @@ -1271,20 +1271,20 @@ POSTHOOK: Input: default@union_top PREHOOK: query: -- create view explain create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: CREATEVIEW POSTHOOK: query: -- create view explain create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATEVIEW STAGE DEPENDENCIES: Stage-0 is a root stage @@ -1295,34 +1295,34 @@ STAGE PLANS: Create View or replace: false columns: key string, value int - expanded text: select `src`.`key`, 0 as `value` from `default`.`src` where `src`.`key` % 3 == 0 limit 3 + expanded text: select `a`.`key`, `a`.`value` from (select `src`.`key`, 0 as `value` from `default`.`src` where `src`.`key` % 3 == 0 limit 3)`a` union all -select `src`.`key`, 1 as `value` from `default`.`src` where `src`.`key` % 3 == 1 limit 3 +select `b`.`key`, `b`.`value` from (select `src`.`key`, 1 as `value` from `default`.`src` where `src`.`key` % 3 == 1 limit 3)`b` union all -select `src`.`key`, 2 as `value` from `default`.`src` where `src`.`key` % 3 == 2 limit 3 +select `c`.`key`, `c`.`value` from (select `src`.`key`, 2 as `value` from `default`.`src` where `src`.`key` % 3 == 2 limit 3)`c` name: default.union_top_view - original text: select key, 0 as value from src where key % 3 == 0 limit 3 + original text: select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: query: create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c PREHOOK: type: CREATEVIEW PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@union_top_view POSTHOOK: query: create view union_top_view as -select key, 0 as value from src where key % 3 == 0 limit 3 +select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a union all -select key, 1 as value from src where key % 3 == 1 limit 3 +select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all -select key, 2 as value from src where key % 3 == 2 limit 3 +select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default diff --git a/ql/src/test/results/clientpositive/varchar_union1.q.out b/ql/src/test/results/clientpositive/varchar_union1.q.out index ac4e86c..9a67de3 100644 --- a/ql/src/test/results/clientpositive/varchar_union1.q.out +++ b/ql/src/test/results/clientpositive/varchar_union1.q.out @@ -95,9 +95,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@varchar_union1_vc1 #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: -- union varchar with different length varchar select * from ( select * from varchar_union1_vc1 @@ -119,9 +116,6 @@ POSTHOOK: Input: default@varchar_union1_vc1 POSTHOOK: Input: default@varchar_union1_vc2 #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: -- union varchar with string select * from ( select * from varchar_union1_vc1 @@ -143,9 +137,6 @@ POSTHOOK: Input: default@varchar_union1_str POSTHOOK: Input: default@varchar_union1_vc1 #### A masked pattern was here #### 1 abc -1 abc -2 abc -3 abc PREHOOK: query: drop table varchar_union1_vc1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@varchar_union1_vc1 diff --git a/ql/src/test/results/clientpositive/vector_multi_insert.q.out b/ql/src/test/results/clientpositive/vector_multi_insert.q.out index 36ae859..2fcacad 100644 --- a/ql/src/test/results/clientpositive/vector_multi_insert.q.out +++ b/ql/src/test/results/clientpositive/vector_multi_insert.q.out @@ -5,11 +5,11 @@ PREHOOK: query: create table orc1 select rn from ( - select cast(1 as int) as rn from src limit 1 + select * from (select cast(1 as int) as rn from src limit 1)a union all - select cast(100 as int) as rn from src limit 1 + select * from (select cast(100 as int) as rn from src limit 1)b union all - select cast(10000 as int) as rn from src limit 1 + select * from (select cast(10000 as int) as rn from src limit 1)c ) t PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -22,11 +22,11 @@ POSTHOOK: query: create table orc1 select rn from ( - select cast(1 as int) as rn from src limit 1 + select * from (select cast(1 as int) as rn from src limit 1)a union all - select cast(100 as int) as rn from src limit 1 + select * from (select cast(100 as int) as rn from src limit 1)b union all - select cast(10000 as int) as rn from src limit 1 + select * from (select cast(10000 as int) as rn from src limit 1)c ) t POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src