diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1d6e6fe..01db782 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7042,12 +7042,15 @@ private Operator createCommonReduceSink(QB qb, Operator input) List> inputOperators = new ArrayList>(ks.size()); List> sprayKeyLists = new ArrayList>(ks.size()); + List> distinctKeyLists = new ArrayList>(ks.size()); // Iterate over each clause for (String dest : ks) { Operator input = inputs.get(dest); RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - List sprayKeys = getDistinctExprs(qbp, dest, inputRR); + + List distinctKeys = getDistinctExprs(qbp, dest, inputRR); + List sprayKeys = new ArrayList(); // Add the group by expressions List grpByExprs = getGroupByForClause(qbp, dest); @@ -7064,10 +7067,43 @@ private Operator createCommonReduceSink(QB qb, Operator input) if (!input.equals(inputOperators.get(i))) { continue; } - if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) { - continue; + + if (distinctKeys.isEmpty()) { + // current dest has no distinct keys. + List combinedList = new ArrayList(); + combineExprNodeLists(sprayKeyLists.get(i), distinctKeyLists.get(i), combinedList); + if (!matchExprLists(combinedList, sprayKeys)) { + continue; + } // else do the common code at the end. + } else { + if (distinctKeyLists.get(i).isEmpty()) { + List combinedList = new ArrayList(); + combineExprNodeLists(sprayKeys, distinctKeys, combinedList); + if (!matchExprLists(combinedList, sprayKeyLists.get(i))) { + continue; + } else { + // we have found a match. insert this distinct clause to head. + distinctKeyLists.remove(i); + sprayKeyLists.remove(i); + distinctKeyLists.add(i, distinctKeys); + sprayKeyLists.add(i, sprayKeys); + commonGroupByDestGroups.get(i).add(0, dest); + found = true; + break; + } + } else { + if (!matchExprLists(distinctKeyLists.get(i), distinctKeys)) { + continue; + } + + if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) { + continue; + } + // else do common code + } } + // common code // A match was found, so add the clause to the corresponding list commonGroupByDestGroups.get(i).add(dest); found = true; @@ -7078,6 +7114,7 @@ private Operator createCommonReduceSink(QB qb, Operator input) if (!found) { inputOperators.add(input); sprayKeyLists.add(sprayKeys); + distinctKeyLists.add(distinctKeys); List destGroup = new ArrayList(); destGroup.add(dest); commonGroupByDestGroups.add(destGroup); @@ -7087,6 +7124,16 @@ private Operator createCommonReduceSink(QB qb, Operator input) return commonGroupByDestGroups; } + private void combineExprNodeLists(List list, List list2, + List combinedList) { + combinedList.addAll(list); + for (ExprNodeDesc elem : list2) { + if (!combinedList.contains(elem)) { + combinedList.add(elem); + } + } + } + // Returns whether or not two lists contain the same elements independent of order private boolean matchExprLists(List list1, List list2) { diff --git ql/src/test/queries/clientpositive/multi_insert_gby3.q ql/src/test/queries/clientpositive/multi_insert_gby3.q new file mode 100644 index 0000000..040adca --- /dev/null +++ ql/src/test/queries/clientpositive/multi_insert_gby3.q @@ -0,0 +1,52 @@ +create table e1 (key string, keyD double); +create table e2 (key string, keyD double, value string); +create table e3 (key string, keyD double); + +explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value; + +explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key; + +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value; + +select * from e1; +select * from e2; + +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key; + +select * from e1; +select * from e2; + +explain +from src +insert overwrite table e1 +select key, count(distinct value) group by key +insert overwrite table e3 +select value, count(distinct key) group by value; + + +explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT overwrite table e3 + SELECT key, COUNT(distinct keyD) group by key, keyD, value; diff --git ql/src/test/results/clientpositive/multi_insert_gby3.q.out ql/src/test/results/clientpositive/multi_insert_gby3.q.out new file mode 100644 index 0000000..a473b3d --- /dev/null +++ ql/src/test/results/clientpositive/multi_insert_gby3.q.out @@ -0,0 +1,2152 @@ +PREHOOK: query: create table e1 (key string, keyD double) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table e1 (key string, keyD double) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@e1 +PREHOOK: query: create table e2 (key string, keyD double, value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table e2 (key string, keyD double, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@e2 +PREHOOK: query: create table e3 (key string, keyD double) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table e3 (key string, keyD double) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@e3 +PREHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION TOK_DOUBLE (TOK_TABLE_OR_COL key)) keyD) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e1))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e2))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL keyD))) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) + +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + a:src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: UDFToDouble(key) + type: double + expr: value + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + Reduce Operator Tree: + Extract + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col2 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + tag: -1 + value expressions: + expr: _col1 + type: double + Reduce Operator Tree: + Forward + Group By Operator + aggregations: + expr: count(DISTINCT KEY._col1:0._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col1) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + Group By Operator + aggregations: + expr: sum(VALUE._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + expr: KEY._col1:0._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col2 + type: double + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-5 + Stats-Aggr Operator + + +PREHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION TOK_DOUBLE (TOK_TABLE_OR_COL key)) keyD) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e2))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL keyD))) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e1))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) + +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + a:src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: UDFToDouble(key) + type: double + expr: value + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + Reduce Operator Tree: + Extract + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col2 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + tag: -1 + value expressions: + expr: _col1 + type: double + Reduce Operator Tree: + Forward + Group By Operator + aggregations: + expr: count(DISTINCT KEY._col1:0._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col1) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + Group By Operator + aggregations: + expr: sum(VALUE._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + expr: KEY._col1:0._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col2 + type: double + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + + Stage: Stage-4 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-5 + Stats-Aggr Operator + + +PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@e1 +PREHOOK: Output: default@e2 +POSTHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@e1 +POSTHOOK: Output: default@e2 +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from e1 +PREHOOK: type: QUERY +PREHOOK: Input: default@e1 +#### A masked pattern was here #### +POSTHOOK: query: select * from e1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@e1 +#### A masked pattern was here #### +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 1.0 +10 1.0 +100 1.0 +103 1.0 +104 1.0 +105 1.0 +11 1.0 +111 1.0 +113 1.0 +114 1.0 +116 1.0 +118 1.0 +119 1.0 +12 1.0 +120 1.0 +125 1.0 +126 1.0 +128 1.0 +129 1.0 +131 1.0 +133 1.0 +134 1.0 +136 1.0 +137 1.0 +138 1.0 +143 1.0 +145 1.0 +146 1.0 +149 1.0 +15 1.0 +150 1.0 +152 1.0 +153 1.0 +155 1.0 +156 1.0 +157 1.0 +158 1.0 +160 1.0 +162 1.0 +163 1.0 +164 1.0 +165 1.0 +166 1.0 +167 1.0 +168 1.0 +169 1.0 +17 1.0 +170 1.0 +172 1.0 +174 1.0 +175 1.0 +176 1.0 +177 1.0 +178 1.0 +179 1.0 +18 1.0 +180 1.0 +181 1.0 +183 1.0 +186 1.0 +187 1.0 +189 1.0 +19 1.0 +190 1.0 +191 1.0 +192 1.0 +193 1.0 +194 1.0 +195 1.0 +196 1.0 +197 1.0 +199 1.0 +2 1.0 +20 1.0 +200 1.0 +201 1.0 +202 1.0 +203 1.0 +205 1.0 +207 1.0 +208 1.0 +209 1.0 +213 1.0 +214 1.0 +216 1.0 +217 1.0 +218 1.0 +219 1.0 +221 1.0 +222 1.0 +223 1.0 +224 1.0 +226 1.0 +228 1.0 +229 1.0 +230 1.0 +233 1.0 +235 1.0 +237 1.0 +238 1.0 +239 1.0 +24 1.0 +241 1.0 +242 1.0 +244 1.0 +247 1.0 +248 1.0 +249 1.0 +252 1.0 +255 1.0 +256 1.0 +257 1.0 +258 1.0 +26 1.0 +260 1.0 +262 1.0 +263 1.0 +265 1.0 +266 1.0 +27 1.0 +272 1.0 +273 1.0 +274 1.0 +275 1.0 +277 1.0 +278 1.0 +28 1.0 +280 1.0 +281 1.0 +282 1.0 +283 1.0 +284 1.0 +285 1.0 +286 1.0 +287 1.0 +288 1.0 +289 1.0 +291 1.0 +292 1.0 +296 1.0 +298 1.0 +30 1.0 +302 1.0 +305 1.0 +306 1.0 +307 1.0 +308 1.0 +309 1.0 +310 1.0 +311 1.0 +315 1.0 +316 1.0 +317 1.0 +318 1.0 +321 1.0 +322 1.0 +323 1.0 +325 1.0 +327 1.0 +33 1.0 +331 1.0 +332 1.0 +333 1.0 +335 1.0 +336 1.0 +338 1.0 +339 1.0 +34 1.0 +341 1.0 +342 1.0 +344 1.0 +345 1.0 +348 1.0 +35 1.0 +351 1.0 +353 1.0 +356 1.0 +360 1.0 +362 1.0 +364 1.0 +365 1.0 +366 1.0 +367 1.0 +368 1.0 +369 1.0 +37 1.0 +373 1.0 +374 1.0 +375 1.0 +377 1.0 +378 1.0 +379 1.0 +382 1.0 +384 1.0 +386 1.0 +389 1.0 +392 1.0 +393 1.0 +394 1.0 +395 1.0 +396 1.0 +397 1.0 +399 1.0 +4 1.0 +400 1.0 +401 1.0 +402 1.0 +403 1.0 +404 1.0 +406 1.0 +407 1.0 +409 1.0 +41 1.0 +411 1.0 +413 1.0 +414 1.0 +417 1.0 +418 1.0 +419 1.0 +42 1.0 +421 1.0 +424 1.0 +427 1.0 +429 1.0 +43 1.0 +430 1.0 +431 1.0 +432 1.0 +435 1.0 +436 1.0 +437 1.0 +438 1.0 +439 1.0 +44 1.0 +443 1.0 +444 1.0 +446 1.0 +448 1.0 +449 1.0 +452 1.0 +453 1.0 +454 1.0 +455 1.0 +457 1.0 +458 1.0 +459 1.0 +460 1.0 +462 1.0 +463 1.0 +466 1.0 +467 1.0 +468 1.0 +469 1.0 +47 1.0 +470 1.0 +472 1.0 +475 1.0 +477 1.0 +478 1.0 +479 1.0 +480 1.0 +481 1.0 +482 1.0 +483 1.0 +484 1.0 +485 1.0 +487 1.0 +489 1.0 +490 1.0 +491 1.0 +492 1.0 +493 1.0 +494 1.0 +495 1.0 +496 1.0 +497 1.0 +498 1.0 +5 1.0 +51 1.0 +53 1.0 +54 1.0 +57 1.0 +58 1.0 +64 1.0 +65 1.0 +66 1.0 +67 1.0 +69 1.0 +70 1.0 +72 1.0 +74 1.0 +76 1.0 +77 1.0 +78 1.0 +8 1.0 +80 1.0 +82 1.0 +83 1.0 +84 1.0 +85 1.0 +86 1.0 +87 1.0 +9 1.0 +90 1.0 +92 1.0 +95 1.0 +96 1.0 +97 1.0 +98 1.0 +PREHOOK: query: select * from e2 +PREHOOK: type: QUERY +PREHOOK: Input: default@e2 +#### A masked pattern was here #### +POSTHOOK: query: select * from e2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@e2 +#### A masked pattern was here #### +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 0.0 val_0 +10 10.0 val_10 +100 200.0 val_100 +103 206.0 val_103 +104 208.0 val_104 +105 105.0 val_105 +11 11.0 val_11 +111 111.0 val_111 +113 226.0 val_113 +114 114.0 val_114 +116 116.0 val_116 +118 236.0 val_118 +119 357.0 val_119 +12 24.0 val_12 +120 240.0 val_120 +125 250.0 val_125 +126 126.0 val_126 +128 384.0 val_128 +129 258.0 val_129 +131 131.0 val_131 +133 133.0 val_133 +134 268.0 val_134 +136 136.0 val_136 +137 274.0 val_137 +138 552.0 val_138 +143 143.0 val_143 +145 145.0 val_145 +146 292.0 val_146 +149 298.0 val_149 +15 30.0 val_15 +150 150.0 val_150 +152 304.0 val_152 +153 153.0 val_153 +155 155.0 val_155 +156 156.0 val_156 +157 157.0 val_157 +158 158.0 val_158 +160 160.0 val_160 +162 162.0 val_162 +163 163.0 val_163 +164 328.0 val_164 +165 330.0 val_165 +166 166.0 val_166 +167 501.0 val_167 +168 168.0 val_168 +169 676.0 val_169 +17 17.0 val_17 +170 170.0 val_170 +172 344.0 val_172 +174 348.0 val_174 +175 350.0 val_175 +176 352.0 val_176 +177 177.0 val_177 +178 178.0 val_178 +179 358.0 val_179 +18 36.0 val_18 +180 180.0 val_180 +181 181.0 val_181 +183 183.0 val_183 +186 186.0 val_186 +187 561.0 val_187 +189 189.0 val_189 +19 19.0 val_19 +190 190.0 val_190 +191 382.0 val_191 +192 192.0 val_192 +193 579.0 val_193 +194 194.0 val_194 +195 390.0 val_195 +196 196.0 val_196 +197 394.0 val_197 +199 597.0 val_199 +2 2.0 val_2 +20 20.0 val_20 +200 400.0 val_200 +201 201.0 val_201 +202 202.0 val_202 +203 406.0 val_203 +205 410.0 val_205 +207 414.0 val_207 +208 624.0 val_208 +209 418.0 val_209 +213 426.0 val_213 +214 214.0 val_214 +216 432.0 val_216 +217 434.0 val_217 +218 218.0 val_218 +219 438.0 val_219 +221 442.0 val_221 +222 222.0 val_222 +223 446.0 val_223 +224 448.0 val_224 +226 226.0 val_226 +228 228.0 val_228 +229 458.0 val_229 +230 1150.0 val_230 +233 466.0 val_233 +235 235.0 val_235 +237 474.0 val_237 +238 476.0 val_238 +239 478.0 val_239 +24 48.0 val_24 +241 241.0 val_241 +242 484.0 val_242 +244 244.0 val_244 +247 247.0 val_247 +248 248.0 val_248 +249 249.0 val_249 +252 252.0 val_252 +255 510.0 val_255 +256 512.0 val_256 +257 257.0 val_257 +258 258.0 val_258 +26 52.0 val_26 +260 260.0 val_260 +262 262.0 val_262 +263 263.0 val_263 +265 530.0 val_265 +266 266.0 val_266 +27 27.0 val_27 +272 544.0 val_272 +273 819.0 val_273 +274 274.0 val_274 +275 275.0 val_275 +277 1108.0 val_277 +278 556.0 val_278 +28 28.0 val_28 +280 560.0 val_280 +281 562.0 val_281 +282 564.0 val_282 +283 283.0 val_283 +284 284.0 val_284 +285 285.0 val_285 +286 286.0 val_286 +287 287.0 val_287 +288 576.0 val_288 +289 289.0 val_289 +291 291.0 val_291 +292 292.0 val_292 +296 296.0 val_296 +298 894.0 val_298 +30 30.0 val_30 +302 302.0 val_302 +305 305.0 val_305 +306 306.0 val_306 +307 614.0 val_307 +308 308.0 val_308 +309 618.0 val_309 +310 310.0 val_310 +311 933.0 val_311 +315 315.0 val_315 +316 948.0 val_316 +317 634.0 val_317 +318 954.0 val_318 +321 642.0 val_321 +322 644.0 val_322 +323 323.0 val_323 +325 650.0 val_325 +327 981.0 val_327 +33 33.0 val_33 +331 662.0 val_331 +332 332.0 val_332 +333 666.0 val_333 +335 335.0 val_335 +336 336.0 val_336 +338 338.0 val_338 +339 339.0 val_339 +34 34.0 val_34 +341 341.0 val_341 +342 684.0 val_342 +344 688.0 val_344 +345 345.0 val_345 +348 1740.0 val_348 +35 105.0 val_35 +351 351.0 val_351 +353 706.0 val_353 +356 356.0 val_356 +360 360.0 val_360 +362 362.0 val_362 +364 364.0 val_364 +365 365.0 val_365 +366 366.0 val_366 +367 734.0 val_367 +368 368.0 val_368 +369 1107.0 val_369 +37 74.0 val_37 +373 373.0 val_373 +374 374.0 val_374 +375 375.0 val_375 +377 377.0 val_377 +378 378.0 val_378 +379 379.0 val_379 +382 764.0 val_382 +384 1152.0 val_384 +386 386.0 val_386 +389 389.0 val_389 +392 392.0 val_392 +393 393.0 val_393 +394 394.0 val_394 +395 790.0 val_395 +396 1188.0 val_396 +397 794.0 val_397 +399 798.0 val_399 +4 4.0 val_4 +400 400.0 val_400 +401 2005.0 val_401 +402 402.0 val_402 +403 1209.0 val_403 +404 808.0 val_404 +406 1624.0 val_406 +407 407.0 val_407 +409 1227.0 val_409 +41 41.0 val_41 +411 411.0 val_411 +413 826.0 val_413 +414 828.0 val_414 +417 1251.0 val_417 +418 418.0 val_418 +419 419.0 val_419 +42 84.0 val_42 +421 421.0 val_421 +424 848.0 val_424 +427 427.0 val_427 +429 858.0 val_429 +43 43.0 val_43 +430 1290.0 val_430 +431 1293.0 val_431 +432 432.0 val_432 +435 435.0 val_435 +436 436.0 val_436 +437 437.0 val_437 +438 1314.0 val_438 +439 878.0 val_439 +44 44.0 val_44 +443 443.0 val_443 +444 444.0 val_444 +446 446.0 val_446 +448 448.0 val_448 +449 449.0 val_449 +452 452.0 val_452 +453 453.0 val_453 +454 1362.0 val_454 +455 455.0 val_455 +457 457.0 val_457 +458 916.0 val_458 +459 918.0 val_459 +460 460.0 val_460 +462 924.0 val_462 +463 926.0 val_463 +466 1398.0 val_466 +467 467.0 val_467 +468 1872.0 val_468 +469 2345.0 val_469 +47 47.0 val_47 +470 470.0 val_470 +472 472.0 val_472 +475 475.0 val_475 +477 477.0 val_477 +478 956.0 val_478 +479 479.0 val_479 +480 1440.0 val_480 +481 481.0 val_481 +482 482.0 val_482 +483 483.0 val_483 +484 484.0 val_484 +485 485.0 val_485 +487 487.0 val_487 +489 1956.0 val_489 +490 490.0 val_490 +491 491.0 val_491 +492 984.0 val_492 +493 493.0 val_493 +494 494.0 val_494 +495 495.0 val_495 +496 496.0 val_496 +497 497.0 val_497 +498 1494.0 val_498 +5 15.0 val_5 +51 102.0 val_51 +53 53.0 val_53 +54 54.0 val_54 +57 57.0 val_57 +58 116.0 val_58 +64 64.0 val_64 +65 65.0 val_65 +66 66.0 val_66 +67 134.0 val_67 +69 69.0 val_69 +70 210.0 val_70 +72 144.0 val_72 +74 74.0 val_74 +76 152.0 val_76 +77 77.0 val_77 +78 78.0 val_78 +8 8.0 val_8 +80 80.0 val_80 +82 82.0 val_82 +83 166.0 val_83 +84 168.0 val_84 +85 85.0 val_85 +86 86.0 val_86 +87 87.0 val_87 +9 9.0 val_9 +90 270.0 val_90 +92 92.0 val_92 +95 190.0 val_95 +96 96.0 val_96 +97 194.0 val_97 +98 196.0 val_98 +PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@e1 +PREHOOK: Output: default@e2 +POSTHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@e1 +POSTHOOK: Output: default@e2 +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from e1 +PREHOOK: type: QUERY +PREHOOK: Input: default@e1 +#### A masked pattern was here #### +POSTHOOK: query: select * from e1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@e1 +#### A masked pattern was here #### +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 1.0 +10 1.0 +100 1.0 +103 1.0 +104 1.0 +105 1.0 +11 1.0 +111 1.0 +113 1.0 +114 1.0 +116 1.0 +118 1.0 +119 1.0 +12 1.0 +120 1.0 +125 1.0 +126 1.0 +128 1.0 +129 1.0 +131 1.0 +133 1.0 +134 1.0 +136 1.0 +137 1.0 +138 1.0 +143 1.0 +145 1.0 +146 1.0 +149 1.0 +15 1.0 +150 1.0 +152 1.0 +153 1.0 +155 1.0 +156 1.0 +157 1.0 +158 1.0 +160 1.0 +162 1.0 +163 1.0 +164 1.0 +165 1.0 +166 1.0 +167 1.0 +168 1.0 +169 1.0 +17 1.0 +170 1.0 +172 1.0 +174 1.0 +175 1.0 +176 1.0 +177 1.0 +178 1.0 +179 1.0 +18 1.0 +180 1.0 +181 1.0 +183 1.0 +186 1.0 +187 1.0 +189 1.0 +19 1.0 +190 1.0 +191 1.0 +192 1.0 +193 1.0 +194 1.0 +195 1.0 +196 1.0 +197 1.0 +199 1.0 +2 1.0 +20 1.0 +200 1.0 +201 1.0 +202 1.0 +203 1.0 +205 1.0 +207 1.0 +208 1.0 +209 1.0 +213 1.0 +214 1.0 +216 1.0 +217 1.0 +218 1.0 +219 1.0 +221 1.0 +222 1.0 +223 1.0 +224 1.0 +226 1.0 +228 1.0 +229 1.0 +230 1.0 +233 1.0 +235 1.0 +237 1.0 +238 1.0 +239 1.0 +24 1.0 +241 1.0 +242 1.0 +244 1.0 +247 1.0 +248 1.0 +249 1.0 +252 1.0 +255 1.0 +256 1.0 +257 1.0 +258 1.0 +26 1.0 +260 1.0 +262 1.0 +263 1.0 +265 1.0 +266 1.0 +27 1.0 +272 1.0 +273 1.0 +274 1.0 +275 1.0 +277 1.0 +278 1.0 +28 1.0 +280 1.0 +281 1.0 +282 1.0 +283 1.0 +284 1.0 +285 1.0 +286 1.0 +287 1.0 +288 1.0 +289 1.0 +291 1.0 +292 1.0 +296 1.0 +298 1.0 +30 1.0 +302 1.0 +305 1.0 +306 1.0 +307 1.0 +308 1.0 +309 1.0 +310 1.0 +311 1.0 +315 1.0 +316 1.0 +317 1.0 +318 1.0 +321 1.0 +322 1.0 +323 1.0 +325 1.0 +327 1.0 +33 1.0 +331 1.0 +332 1.0 +333 1.0 +335 1.0 +336 1.0 +338 1.0 +339 1.0 +34 1.0 +341 1.0 +342 1.0 +344 1.0 +345 1.0 +348 1.0 +35 1.0 +351 1.0 +353 1.0 +356 1.0 +360 1.0 +362 1.0 +364 1.0 +365 1.0 +366 1.0 +367 1.0 +368 1.0 +369 1.0 +37 1.0 +373 1.0 +374 1.0 +375 1.0 +377 1.0 +378 1.0 +379 1.0 +382 1.0 +384 1.0 +386 1.0 +389 1.0 +392 1.0 +393 1.0 +394 1.0 +395 1.0 +396 1.0 +397 1.0 +399 1.0 +4 1.0 +400 1.0 +401 1.0 +402 1.0 +403 1.0 +404 1.0 +406 1.0 +407 1.0 +409 1.0 +41 1.0 +411 1.0 +413 1.0 +414 1.0 +417 1.0 +418 1.0 +419 1.0 +42 1.0 +421 1.0 +424 1.0 +427 1.0 +429 1.0 +43 1.0 +430 1.0 +431 1.0 +432 1.0 +435 1.0 +436 1.0 +437 1.0 +438 1.0 +439 1.0 +44 1.0 +443 1.0 +444 1.0 +446 1.0 +448 1.0 +449 1.0 +452 1.0 +453 1.0 +454 1.0 +455 1.0 +457 1.0 +458 1.0 +459 1.0 +460 1.0 +462 1.0 +463 1.0 +466 1.0 +467 1.0 +468 1.0 +469 1.0 +47 1.0 +470 1.0 +472 1.0 +475 1.0 +477 1.0 +478 1.0 +479 1.0 +480 1.0 +481 1.0 +482 1.0 +483 1.0 +484 1.0 +485 1.0 +487 1.0 +489 1.0 +490 1.0 +491 1.0 +492 1.0 +493 1.0 +494 1.0 +495 1.0 +496 1.0 +497 1.0 +498 1.0 +5 1.0 +51 1.0 +53 1.0 +54 1.0 +57 1.0 +58 1.0 +64 1.0 +65 1.0 +66 1.0 +67 1.0 +69 1.0 +70 1.0 +72 1.0 +74 1.0 +76 1.0 +77 1.0 +78 1.0 +8 1.0 +80 1.0 +82 1.0 +83 1.0 +84 1.0 +85 1.0 +86 1.0 +87 1.0 +9 1.0 +90 1.0 +92 1.0 +95 1.0 +96 1.0 +97 1.0 +98 1.0 +PREHOOK: query: select * from e2 +PREHOOK: type: QUERY +PREHOOK: Input: default@e2 +#### A masked pattern was here #### +POSTHOOK: query: select * from e2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@e2 +#### A masked pattern was here #### +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 0.0 val_0 +10 10.0 val_10 +100 200.0 val_100 +103 206.0 val_103 +104 208.0 val_104 +105 105.0 val_105 +11 11.0 val_11 +111 111.0 val_111 +113 226.0 val_113 +114 114.0 val_114 +116 116.0 val_116 +118 236.0 val_118 +119 357.0 val_119 +12 24.0 val_12 +120 240.0 val_120 +125 250.0 val_125 +126 126.0 val_126 +128 384.0 val_128 +129 258.0 val_129 +131 131.0 val_131 +133 133.0 val_133 +134 268.0 val_134 +136 136.0 val_136 +137 274.0 val_137 +138 552.0 val_138 +143 143.0 val_143 +145 145.0 val_145 +146 292.0 val_146 +149 298.0 val_149 +15 30.0 val_15 +150 150.0 val_150 +152 304.0 val_152 +153 153.0 val_153 +155 155.0 val_155 +156 156.0 val_156 +157 157.0 val_157 +158 158.0 val_158 +160 160.0 val_160 +162 162.0 val_162 +163 163.0 val_163 +164 328.0 val_164 +165 330.0 val_165 +166 166.0 val_166 +167 501.0 val_167 +168 168.0 val_168 +169 676.0 val_169 +17 17.0 val_17 +170 170.0 val_170 +172 344.0 val_172 +174 348.0 val_174 +175 350.0 val_175 +176 352.0 val_176 +177 177.0 val_177 +178 178.0 val_178 +179 358.0 val_179 +18 36.0 val_18 +180 180.0 val_180 +181 181.0 val_181 +183 183.0 val_183 +186 186.0 val_186 +187 561.0 val_187 +189 189.0 val_189 +19 19.0 val_19 +190 190.0 val_190 +191 382.0 val_191 +192 192.0 val_192 +193 579.0 val_193 +194 194.0 val_194 +195 390.0 val_195 +196 196.0 val_196 +197 394.0 val_197 +199 597.0 val_199 +2 2.0 val_2 +20 20.0 val_20 +200 400.0 val_200 +201 201.0 val_201 +202 202.0 val_202 +203 406.0 val_203 +205 410.0 val_205 +207 414.0 val_207 +208 624.0 val_208 +209 418.0 val_209 +213 426.0 val_213 +214 214.0 val_214 +216 432.0 val_216 +217 434.0 val_217 +218 218.0 val_218 +219 438.0 val_219 +221 442.0 val_221 +222 222.0 val_222 +223 446.0 val_223 +224 448.0 val_224 +226 226.0 val_226 +228 228.0 val_228 +229 458.0 val_229 +230 1150.0 val_230 +233 466.0 val_233 +235 235.0 val_235 +237 474.0 val_237 +238 476.0 val_238 +239 478.0 val_239 +24 48.0 val_24 +241 241.0 val_241 +242 484.0 val_242 +244 244.0 val_244 +247 247.0 val_247 +248 248.0 val_248 +249 249.0 val_249 +252 252.0 val_252 +255 510.0 val_255 +256 512.0 val_256 +257 257.0 val_257 +258 258.0 val_258 +26 52.0 val_26 +260 260.0 val_260 +262 262.0 val_262 +263 263.0 val_263 +265 530.0 val_265 +266 266.0 val_266 +27 27.0 val_27 +272 544.0 val_272 +273 819.0 val_273 +274 274.0 val_274 +275 275.0 val_275 +277 1108.0 val_277 +278 556.0 val_278 +28 28.0 val_28 +280 560.0 val_280 +281 562.0 val_281 +282 564.0 val_282 +283 283.0 val_283 +284 284.0 val_284 +285 285.0 val_285 +286 286.0 val_286 +287 287.0 val_287 +288 576.0 val_288 +289 289.0 val_289 +291 291.0 val_291 +292 292.0 val_292 +296 296.0 val_296 +298 894.0 val_298 +30 30.0 val_30 +302 302.0 val_302 +305 305.0 val_305 +306 306.0 val_306 +307 614.0 val_307 +308 308.0 val_308 +309 618.0 val_309 +310 310.0 val_310 +311 933.0 val_311 +315 315.0 val_315 +316 948.0 val_316 +317 634.0 val_317 +318 954.0 val_318 +321 642.0 val_321 +322 644.0 val_322 +323 323.0 val_323 +325 650.0 val_325 +327 981.0 val_327 +33 33.0 val_33 +331 662.0 val_331 +332 332.0 val_332 +333 666.0 val_333 +335 335.0 val_335 +336 336.0 val_336 +338 338.0 val_338 +339 339.0 val_339 +34 34.0 val_34 +341 341.0 val_341 +342 684.0 val_342 +344 688.0 val_344 +345 345.0 val_345 +348 1740.0 val_348 +35 105.0 val_35 +351 351.0 val_351 +353 706.0 val_353 +356 356.0 val_356 +360 360.0 val_360 +362 362.0 val_362 +364 364.0 val_364 +365 365.0 val_365 +366 366.0 val_366 +367 734.0 val_367 +368 368.0 val_368 +369 1107.0 val_369 +37 74.0 val_37 +373 373.0 val_373 +374 374.0 val_374 +375 375.0 val_375 +377 377.0 val_377 +378 378.0 val_378 +379 379.0 val_379 +382 764.0 val_382 +384 1152.0 val_384 +386 386.0 val_386 +389 389.0 val_389 +392 392.0 val_392 +393 393.0 val_393 +394 394.0 val_394 +395 790.0 val_395 +396 1188.0 val_396 +397 794.0 val_397 +399 798.0 val_399 +4 4.0 val_4 +400 400.0 val_400 +401 2005.0 val_401 +402 402.0 val_402 +403 1209.0 val_403 +404 808.0 val_404 +406 1624.0 val_406 +407 407.0 val_407 +409 1227.0 val_409 +41 41.0 val_41 +411 411.0 val_411 +413 826.0 val_413 +414 828.0 val_414 +417 1251.0 val_417 +418 418.0 val_418 +419 419.0 val_419 +42 84.0 val_42 +421 421.0 val_421 +424 848.0 val_424 +427 427.0 val_427 +429 858.0 val_429 +43 43.0 val_43 +430 1290.0 val_430 +431 1293.0 val_431 +432 432.0 val_432 +435 435.0 val_435 +436 436.0 val_436 +437 437.0 val_437 +438 1314.0 val_438 +439 878.0 val_439 +44 44.0 val_44 +443 443.0 val_443 +444 444.0 val_444 +446 446.0 val_446 +448 448.0 val_448 +449 449.0 val_449 +452 452.0 val_452 +453 453.0 val_453 +454 1362.0 val_454 +455 455.0 val_455 +457 457.0 val_457 +458 916.0 val_458 +459 918.0 val_459 +460 460.0 val_460 +462 924.0 val_462 +463 926.0 val_463 +466 1398.0 val_466 +467 467.0 val_467 +468 1872.0 val_468 +469 2345.0 val_469 +47 47.0 val_47 +470 470.0 val_470 +472 472.0 val_472 +475 475.0 val_475 +477 477.0 val_477 +478 956.0 val_478 +479 479.0 val_479 +480 1440.0 val_480 +481 481.0 val_481 +482 482.0 val_482 +483 483.0 val_483 +484 484.0 val_484 +485 485.0 val_485 +487 487.0 val_487 +489 1956.0 val_489 +490 490.0 val_490 +491 491.0 val_491 +492 984.0 val_492 +493 493.0 val_493 +494 494.0 val_494 +495 495.0 val_495 +496 496.0 val_496 +497 497.0 val_497 +498 1494.0 val_498 +5 15.0 val_5 +51 102.0 val_51 +53 53.0 val_53 +54 54.0 val_54 +57 57.0 val_57 +58 116.0 val_58 +64 64.0 val_64 +65 65.0 val_65 +66 66.0 val_66 +67 134.0 val_67 +69 69.0 val_69 +70 210.0 val_70 +72 144.0 val_72 +74 74.0 val_74 +76 152.0 val_76 +77 77.0 val_77 +78 78.0 val_78 +8 8.0 val_8 +80 80.0 val_80 +82 82.0 val_82 +83 166.0 val_83 +84 168.0 val_84 +85 85.0 val_85 +86 86.0 val_86 +87 87.0 val_87 +9 9.0 val_9 +90 270.0 val_90 +92 92.0 val_92 +95 190.0 val_95 +96 96.0 val_96 +97 194.0 val_97 +98 196.0 val_98 +PREHOOK: query: explain +from src +insert overwrite table e1 +select key, count(distinct value) group by key +insert overwrite table e3 +select value, count(distinct key) group by value +PREHOOK: type: QUERY +POSTHOOK: query: explain +from src +insert overwrite table e1 +select key, count(distinct value) group by key +insert overwrite table e3 +select value, count(distinct key) group by value +POSTHOOK: type: QUERY +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e1))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e3))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value)))) + +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: key, value + Group By Operator + aggregations: + expr: count(DISTINCT value) + bucketGroup: false + keys: + expr: key + type: string + expr: value + type: string + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + tag: -1 + value expressions: + expr: _col2 + type: bigint + Select Operator + expressions: + expr: value + type: string + expr: key + type: string + outputColumnNames: value, key + Group By Operator + aggregations: + expr: count(DISTINCT key) + bucketGroup: false + keys: + expr: value + type: string + expr: key + type: string + mode: hash + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY._col1:0._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + mode: mergepartial + outputColumnNames: _col0, _col1 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col1) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + tag: -1 + value expressions: + expr: _col2 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(DISTINCT KEY._col1:0._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + mode: mergepartial + outputColumnNames: _col0, _col1 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col1) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e3 + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e3 + + Stage: Stage-5 + Stats-Aggr Operator + + +PREHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT overwrite table e3 + SELECT key, COUNT(distinct keyD) group by key, keyD, value +PREHOOK: type: QUERY +POSTHOOK: query: explain +FROM (select key, cast(key as double) as keyD, value from src order by key) a +INSERT OVERWRITE TABLE e1 + SELECT key, COUNT(distinct value) group by key +INSERT OVERWRITE TABLE e2 + SELECT key, sum(keyD), value group by key, value +INSERT overwrite table e3 + SELECT key, COUNT(distinct keyD) group by key, keyD, value +POSTHOOK: type: QUERY +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e1.keyd EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.keyd EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: e2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION TOK_DOUBLE (TOK_TABLE_OR_COL key)) keyD) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e1))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e2))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_TABLE_OR_COL keyD))) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME e3))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI COUNT (TOK_TABLE_OR_COL keyD)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL keyD) (TOK_TABLE_OR_COL value)))) + +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-4 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + a:src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: UDFToDouble(key) + type: double + expr: value + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + sort order: ++++ + Map-reduce partition columns: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + Reduce Operator Tree: + Extract + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Group By Operator + aggregations: + expr: count(DISTINCT _col1) + bucketGroup: false + keys: + expr: _col0 + type: string + expr: _col1 + type: double + expr: _col2 + type: string + mode: complete + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col3) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 3 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e3 + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col2 + type: string + sort order: ++ + Map-reduce partition columns: + expr: _col0 + type: string + tag: -1 + value expressions: + expr: _col1 + type: double + Reduce Operator Tree: + Forward + Group By Operator + aggregations: + expr: count(DISTINCT KEY._col1:0._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1 + Select Operator + expressions: + expr: _col0 + type: string + expr: UDFToDouble(_col1) + type: double + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + Group By Operator + aggregations: + expr: sum(VALUE._col0) + bucketGroup: false + keys: + expr: KEY._col0 + type: string + expr: KEY._col1:0._col0 + type: string + mode: complete + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col2 + type: double + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e1 + + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e2 + + Stage: Stage-6 + Stats-Aggr Operator + + Stage: Stage-2 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.e3 + + Stage: Stage-7 + Stats-Aggr Operator + +