diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 88ef4f8..d69acd3 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -453,4 +453,4 @@ spark.query.files=alter_merge_orc.q \ union_remove_6.q \ union_remove_7.q \ union_remove_8.q \ - union_remove_9.q + union_remove_9.q \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 9c808d4..daaa377 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1744,6 +1744,12 @@ public static Path createMoveTask(Task currTask, boolean } } else { fileSinkDesc.setDirName(tmpDir); + if (fileSinkDesc.getLinkedFileSinkDesc() != null) { + for (FileSinkDesc fsConf:fileSinkDesc.getLinkedFileSinkDesc()) { + fsConf.setParentDir(tmpDir); + fsConf.setDirName(tmpDir); + } + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index 5ddc16d..4ab7300 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -132,6 +132,10 @@ // remember which reducesinks we've already connected public final Set connectedReduceSinks; + //If many fileSinkDescs are linked to each other, it is a good idea to keep track of + // tasks for first fileSinkDesc. others can use it + private Map> linkedFileDescTasks; + @SuppressWarnings("unchecked") public GenSparkProcContext(HiveConf conf, ParseContext parseContext, List> moveTask, List> rootTasks, @@ -166,4 +170,13 @@ public GenSparkProcContext(HiveConf conf, ParseContext parseContext, rootTasks.add(currentTask); } + public Map> getLinkedFileDescTasks() { + return linkedFileDescTasks; + } + + public void setLinkedFileDescTasks( + Map> linkedFileDescTasks) { + this.linkedFileDescTasks = linkedFileDescTasks; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 379a39c..fed0b74 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse.spark; +import java.io.Serializable; import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; @@ -29,6 +30,7 @@ import java.util.Set; import com.google.common.base.Strings; + import org.apache.hadoop.fs.Path; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -273,7 +275,15 @@ public void processFileSink(GenSparkProcContext context, FileSinkOperator fileSi boolean isInsertTable = // is INSERT OVERWRITE TABLE GenMapRedUtils.isInsertInto(parseContext, fileSink); HiveConf hconf = parseContext.getConf(); - + // If this file sink desc has been processed due to a linked file sink desc, + // use that task + Map> fileSinkDescs = context.getLinkedFileDescTasks(); + if (fileSinkDescs != null) { + Task task = fileSinkDescs.get(fileSink.getConf()); + if (task != null) { + return; + } + } boolean chDir = GenMapRedUtils.isMergeRequired(context.moveTask, hconf, fileSink, context.currentTask, isInsertTable); @@ -288,7 +298,20 @@ public void processFileSink(GenSparkProcContext context, FileSinkOperator fileSi context.dependencyTask, context.moveTask, hconf, context.currentTask); } - + FileSinkDesc fileSinkDesc = fileSink.getConf(); + if (fileSinkDesc.getLinkedFileSinkDesc() != null) { + Map> linkedFileDescTasks = + context.getLinkedFileDescTasks(); + if (linkedFileDescTasks == null) { + linkedFileDescTasks = new HashMap>(); + context.setLinkedFileDescTasks(linkedFileDescTasks); + } + if (context.currentTask != null) { + for (FileSinkDesc fileDesc : fileSinkDesc.getLinkedFileSinkDesc()) { + linkedFileDescTasks.put(fileDesc, context.currentTask); + } + } + } FetchTask fetchTask = parseContext.getFetchTask(); if (fetchTask != null && context.currentTask.getNumChild() == 0) { if (fetchTask.isFetchFrom(fileSink.getConf())) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 76fc290..74cc394 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -30,6 +31,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.ConditionalTask; @@ -64,6 +66,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TaskCompiler; import org.apache.hadoop.hive.ql.plan.BaseWork; +import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -183,6 +186,46 @@ public Object process(Node n, Stack s, GenSparkUtils.getUtils().removeUnionOperators(conf, procCtx, w); } + // re-construct the linkedfilesinkdesc for all fileSinks + Map> linkedFileSinkDescMap = new HashMap>(); + for (FileSinkOperator fileSink: procCtx.fileSinkSet) { + FileSinkDesc desc = fileSink.getConf(); + Path path = desc.getDirName(); + if (linkedFileSinkDescMap.containsKey(path)) { + Set descSet = linkedFileSinkDescMap.get(path); + List descList = desc.getLinkedFileSinkDesc(); + if (descList != null) { + for (FileSinkDesc fsDesc : descList) { + descSet.add(fsDesc); + } + } + } else { + Set descSet = new HashSet(); + List descList = desc.getLinkedFileSinkDesc(); + if (descList != null) { + for (FileSinkDesc fsDesc : descList) { + descSet.add(fsDesc); + } + linkedFileSinkDescMap.put(path, descSet); + } + } + } + + for (FileSinkOperator fileSink: procCtx.fileSinkSet) { + FileSinkDesc desc = fileSink.getConf(); + Path path = desc.getDirName(); + Set descSet = linkedFileSinkDescMap.get(path); + List linked = null; + if (descSet != null) { + linked = new ArrayList(); + for (FileSinkDesc fsDesc : descSet) { + linked.add(fsDesc); + } + } + if (linked != null) { + desc.setLinkedFileSinkDesc(linked); + } + } // finally make sure the file sink operators are set up right for (FileSinkOperator fileSink: procCtx.fileSinkSet) { GenSparkUtils.getUtils().processFileSink(procCtx, fileSink); diff --git a/ql/src/test/queries/clientpositive/union_remove_1.q b/ql/src/test/queries/clientpositive/union_remove_1.q index c87b3fe..0db1743 100644 --- a/ql/src/test/queries/clientpositive/union_remove_1.q +++ b/ql/src/test/queries/clientpositive/union_remove_1.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_10.q b/ql/src/test/queries/clientpositive/union_remove_10.q index 6701952..1e9c201 100644 --- a/ql/src/test/queries/clientpositive/union_remove_10.q +++ b/ql/src/test/queries/clientpositive/union_remove_10.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_11.q b/ql/src/test/queries/clientpositive/union_remove_11.q index 4b2fa42..7052c69 100644 --- a/ql/src/test/queries/clientpositive/union_remove_11.q +++ b/ql/src/test/queries/clientpositive/union_remove_11.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_12.q b/ql/src/test/queries/clientpositive/union_remove_12.q index 69d0d0a..67a1829 100644 --- a/ql/src/test/queries/clientpositive/union_remove_12.q +++ b/ql/src/test/queries/clientpositive/union_remove_12.q @@ -3,6 +3,7 @@ set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; set hive.auto.convert.join=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_13.q b/ql/src/test/queries/clientpositive/union_remove_13.q index 7605f0e..29c164a 100644 --- a/ql/src/test/queries/clientpositive/union_remove_13.q +++ b/ql/src/test/queries/clientpositive/union_remove_13.q @@ -3,6 +3,7 @@ set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; set hive.auto.convert.join=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_14.q b/ql/src/test/queries/clientpositive/union_remove_14.q index a4fdfc8..ca2f5e5 100644 --- a/ql/src/test/queries/clientpositive/union_remove_14.q +++ b/ql/src/test/queries/clientpositive/union_remove_14.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.auto.convert.join=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_15.q b/ql/src/test/queries/clientpositive/union_remove_15.q index e3c937b..72ced75 100644 --- a/ql/src/test/queries/clientpositive/union_remove_15.q +++ b/ql/src/test/queries/clientpositive/union_remove_15.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; diff --git a/ql/src/test/queries/clientpositive/union_remove_16.q b/ql/src/test/queries/clientpositive/union_remove_16.q index 537078b..72e6cb1 100644 --- a/ql/src/test/queries/clientpositive/union_remove_16.q +++ b/ql/src/test/queries/clientpositive/union_remove_16.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_17.q b/ql/src/test/queries/clientpositive/union_remove_17.q index d70f3d3..fa68755 100644 --- a/ql/src/test/queries/clientpositive/union_remove_17.q +++ b/ql/src/test/queries/clientpositive/union_remove_17.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; diff --git a/ql/src/test/queries/clientpositive/union_remove_18.q b/ql/src/test/queries/clientpositive/union_remove_18.q index 6352bc3..6d2d331 100644 --- a/ql/src/test/queries/clientpositive/union_remove_18.q +++ b/ql/src/test/queries/clientpositive/union_remove_18.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; diff --git a/ql/src/test/queries/clientpositive/union_remove_19.q b/ql/src/test/queries/clientpositive/union_remove_19.q index 8c45953..36a1e9b 100644 --- a/ql/src/test/queries/clientpositive/union_remove_19.q +++ b/ql/src/test/queries/clientpositive/union_remove_19.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_2.q b/ql/src/test/queries/clientpositive/union_remove_2.q index 83cd288..0142325 100644 --- a/ql/src/test/queries/clientpositive/union_remove_2.q +++ b/ql/src/test/queries/clientpositive/union_remove_2.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_20.q b/ql/src/test/queries/clientpositive/union_remove_20.q index f80f7c1..1c59ef2 100644 --- a/ql/src/test/queries/clientpositive/union_remove_20.q +++ b/ql/src/test/queries/clientpositive/union_remove_20.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_21.q b/ql/src/test/queries/clientpositive/union_remove_21.q index 8963c25..cbaa08b 100644 --- a/ql/src/test/queries/clientpositive/union_remove_21.q +++ b/ql/src/test/queries/clientpositive/union_remove_21.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_22.q b/ql/src/test/queries/clientpositive/union_remove_22.q index b0c1ccd..982912b 100644 --- a/ql/src/test/queries/clientpositive/union_remove_22.q +++ b/ql/src/test/queries/clientpositive/union_remove_22.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_23.q b/ql/src/test/queries/clientpositive/union_remove_23.q index a1b989a..63e4418 100644 --- a/ql/src/test/queries/clientpositive/union_remove_23.q +++ b/ql/src/test/queries/clientpositive/union_remove_23.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_24.q b/ql/src/test/queries/clientpositive/union_remove_24.q index ec561e0..88c378d 100644 --- a/ql/src/test/queries/clientpositive/union_remove_24.q +++ b/ql/src/test/queries/clientpositive/union_remove_24.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_25.q b/ql/src/test/queries/clientpositive/union_remove_25.q index 76c1ff5..c6c09e1 100644 --- a/ql/src/test/queries/clientpositive/union_remove_25.q +++ b/ql/src/test/queries/clientpositive/union_remove_25.q @@ -3,6 +3,7 @@ set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_3.q b/ql/src/test/queries/clientpositive/union_remove_3.q index 9617f73..7e1b113 100644 --- a/ql/src/test/queries/clientpositive/union_remove_3.q +++ b/ql/src/test/queries/clientpositive/union_remove_3.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_4.q b/ql/src/test/queries/clientpositive/union_remove_4.q index cae323b..44b31d6 100644 --- a/ql/src/test/queries/clientpositive/union_remove_4.q +++ b/ql/src/test/queries/clientpositive/union_remove_4.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_5.q b/ql/src/test/queries/clientpositive/union_remove_5.q index 5df84e1..c5c0b7f 100644 --- a/ql/src/test/queries/clientpositive/union_remove_5.q +++ b/ql/src/test/queries/clientpositive/union_remove_5.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/queries/clientpositive/union_remove_6.q b/ql/src/test/queries/clientpositive/union_remove_6.q index bfce26d..6990ed2 100644 --- a/ql/src/test/queries/clientpositive/union_remove_6.q +++ b/ql/src/test/queries/clientpositive/union_remove_6.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_7.q b/ql/src/test/queries/clientpositive/union_remove_7.q index 3a95674..c254aba 100644 --- a/ql/src/test/queries/clientpositive/union_remove_7.q +++ b/ql/src/test/queries/clientpositive/union_remove_7.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_8.q b/ql/src/test/queries/clientpositive/union_remove_8.q index a83a43e..8dfb8e8 100644 --- a/ql/src/test/queries/clientpositive/union_remove_8.q +++ b/ql/src/test/queries/clientpositive/union_remove_8.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=false; set hive.merge.mapfiles=false; set hive.merge.mapredfiles=false; set mapred.input.dir.recursive=true; diff --git a/ql/src/test/queries/clientpositive/union_remove_9.q b/ql/src/test/queries/clientpositive/union_remove_9.q index e71f6dd..c9a4dc3 100644 --- a/ql/src/test/queries/clientpositive/union_remove_9.q +++ b/ql/src/test/queries/clientpositive/union_remove_9.q @@ -2,6 +2,7 @@ set hive.stats.autogather=false; set hive.optimize.union.remove=true; set hive.mapred.supports.subdirectories=true; +set hive.merge.sparkfiles=true; set hive.merge.mapfiles=true; set hive.merge.mapredfiles=true; set hive.merge.smallfiles.avgsize=1; diff --git a/ql/src/test/results/clientpositive/spark/union10.q.out b/ql/src/test/results/clientpositive/spark/union10.q.out index 20c681e..2b67840 100644 --- a/ql/src/test/results/clientpositive/spark/union10.q.out +++ b/ql/src/test/results/clientpositive/spark/union10.q.out @@ -31,9 +31,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -169,15 +166,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-6 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION ALL @@ -188,9 +176,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@tmptable [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION ALL diff --git a/ql/src/test/results/clientpositive/spark/union18.q.out b/ql/src/test/results/clientpositive/spark/union18.q.out index 3f37a0a..658efb7 100644 --- a/ql/src/test/results/clientpositive/spark/union18.q.out +++ b/ql/src/test/results/clientpositive/spark/union18.q.out @@ -37,12 +37,8 @@ STAGE DEPENDENCIES: Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 - Stage-7 depends on stages: Stage-1 - Stage-9 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -143,12 +139,6 @@ STAGE PLANS: Stage: Stage-4 Stats-Aggr Operator - Stage: Stage-6 - Stats-Aggr Operator - - Stage: Stage-8 - Stats-Aggr Operator - Stage: Stage-1 Move Operator tables: @@ -162,12 +152,6 @@ STAGE PLANS: Stage: Stage-5 Stats-Aggr Operator - Stage: Stage-7 - Stats-Aggr Operator - - Stage: Stage-9 - Stats-Aggr Operator - PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc @@ -179,10 +163,6 @@ PREHOOK: Output: default@dest1 PREHOOK: Output: default@dest2 [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc diff --git a/ql/src/test/results/clientpositive/spark/union19.q.out b/ql/src/test/results/clientpositive/spark/union19.q.out index 6922fcd..91dc048 100644 --- a/ql/src/test/results/clientpositive/spark/union19.q.out +++ b/ql/src/test/results/clientpositive/spark/union19.q.out @@ -39,8 +39,6 @@ STAGE DEPENDENCIES: Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 - Stage-6 depends on stages: Stage-1 - Stage-7 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -181,12 +179,6 @@ STAGE PLANS: Stage: Stage-5 Stats-Aggr Operator - Stage: Stage-6 - Stats-Aggr Operator - - Stage: Stage-7 - Stats-Aggr Operator - PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc @@ -198,8 +190,6 @@ PREHOOK: Output: default@dest1 PREHOOK: Output: default@dest2 [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL select s2.key as key, s2.value as value from src s2) unionsrc diff --git a/ql/src/test/results/clientpositive/spark/union28.q.out b/ql/src/test/results/clientpositive/spark/union28.q.out index 8bd5218..c55aafb 100644 --- a/ql/src/test/results/clientpositive/spark/union28.q.out +++ b/ql/src/test/results/clientpositive/spark/union28.q.out @@ -37,9 +37,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -174,15 +171,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-6 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src @@ -198,9 +186,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src diff --git a/ql/src/test/results/clientpositive/spark/union29.q.out b/ql/src/test/results/clientpositive/spark/union29.q.out index b9546ef..93c94d5 100644 --- a/ql/src/test/results/clientpositive/spark/union29.q.out +++ b/ql/src/test/results/clientpositive/spark/union29.q.out @@ -37,9 +37,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -124,15 +121,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-6 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src @@ -148,9 +136,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table union_subq_union select * from ( select key, value from src diff --git a/ql/src/test/results/clientpositive/spark/union3.q.out b/ql/src/test/results/clientpositive/spark/union3.q.out index 3ae6536..d238a71 100644 --- a/ql/src/test/results/clientpositive/spark/union3.q.out +++ b/ql/src/test/results/clientpositive/spark/union3.q.out @@ -239,10 +239,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_out [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table union_out SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union30.q.out b/ql/src/test/results/clientpositive/spark/union30.q.out index 12717a1..b1014b6 100644 --- a/ql/src/test/results/clientpositive/spark/union30.q.out +++ b/ql/src/test/results/clientpositive/spark/union30.q.out @@ -51,10 +51,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-0 - Stage-7 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -215,18 +211,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-6 - Stats-Aggr Operator - - Stage: Stage-7 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table union_subq_union select * from ( @@ -249,10 +233,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@union_subq_union [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table union_subq_union select * from ( diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out index b89757f..7eadb04 100644 --- a/ql/src/test/results/clientpositive/spark/union33.q.out +++ b/ql/src/test/results/clientpositive/spark/union33.q.out @@ -37,8 +37,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -142,12 +140,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src @@ -160,8 +152,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@test_src [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, value FROM src @@ -207,8 +197,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -312,12 +300,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src @@ -330,8 +312,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@test_src [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( SELECT key, COUNT(*) AS value FROM src diff --git a/ql/src/test/results/clientpositive/spark/union4.q.out b/ql/src/test/results/clientpositive/spark/union4.q.out index 6341cd9..fd2abc8 100644 --- a/ql/src/test/results/clientpositive/spark/union4.q.out +++ b/ql/src/test/results/clientpositive/spark/union4.q.out @@ -29,8 +29,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -130,12 +128,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION ALL @@ -144,8 +136,6 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@tmptable [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 UNION ALL diff --git a/ql/src/test/results/clientpositive/spark/union6.q.out b/ql/src/test/results/clientpositive/spark/union6.q.out index 263d9f4..5bbb681 100644 --- a/ql/src/test/results/clientpositive/spark/union6.q.out +++ b/ql/src/test/results/clientpositive/spark/union6.q.out @@ -29,8 +29,6 @@ STAGE DEPENDENCIES: Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -111,12 +109,6 @@ STAGE PLANS: Stage: Stage-3 Stats-Aggr Operator - Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-5 - Stats-Aggr Operator - PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL @@ -126,8 +118,6 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@tmptable [Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to -[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to POSTHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out index 927a15d..08f1bf5 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out @@ -78,8 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -167,6 +172,15 @@ STAGE PLANS: Union 3 Vertex: Union 3 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -180,6 +194,22 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-3 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-5 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out index 96651e1..a5b459c 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out @@ -78,8 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -148,6 +153,15 @@ STAGE PLANS: Union 2 Vertex: Union 2 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -161,6 +175,22 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-3 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-5 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out index 0954ae4..bacb52f 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out @@ -66,8 +66,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -160,6 +165,15 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -175,6 +189,22 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-3 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-5 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out index cc46dda..a429f3e 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out @@ -62,8 +62,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -156,6 +161,15 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -169,6 +183,42 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Merge + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-5 + Spark +#### A masked pattern was here #### + Vertices: + Merge + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out index f6cdeb3..c2c7c08 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out @@ -70,8 +70,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -156,6 +161,15 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -169,6 +183,42 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Merge + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-5 + Spark +#### A masked pattern was here #### + Vertices: + Merge + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out index 1f0260c..5e997fd 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out @@ -74,8 +74,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-2 depends on stages: Stage-4, Stage-3, Stage-6 Stage-0 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -163,6 +168,15 @@ STAGE PLANS: Union 2 Vertex: Union 2 + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + Stage: Stage-2 Dependency Collection @@ -176,6 +190,22 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-3 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-5 + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM (