diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/IdentityTran.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/IdentityTran.java deleted file mode 100644 index eb758e0..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/IdentityTran.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.spark; - -import org.apache.hadoop.io.BytesWritable; -import org.apache.spark.api.java.JavaPairRDD; - -public class IdentityTran implements SparkTran { - - @Override - public JavaPairRDD transform(JavaPairRDD input) { - return input; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java index 438efab..fb6335d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java @@ -90,26 +90,10 @@ public SparkPlan generate(SparkWork sparkWork) throws Exception { for (BaseWork work : sparkWork.getAllWork()) { SparkTran tran; - if (work instanceof MapWork) { - SparkTran mapInput = generateParentTran(sparkPlan, sparkWork, work); - tran = generate((MapWork)work); - sparkPlan.addTran(tran); - sparkPlan.connect(mapInput, tran); - } else if (work instanceof ReduceWork) { - SparkTran shuffleTran = generateParentTran(sparkPlan, sparkWork, work); - tran = generate((ReduceWork)work); - sparkPlan.addTran(tran); - sparkPlan.connect(shuffleTran, tran); - } else { - List parentWorks = sparkWork.getParents(work); - tran = new IdentityTran(); - sparkPlan.addTran(tran); - for (BaseWork parentWork : parentWorks) { - SparkTran parentTran = workToTranMap.get(parentWork); - sparkPlan.connect(parentTran, tran); - } - } - + SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work); + tran = generate(work); + sparkPlan.addTran(tran); + sparkPlan.connect(parentTran, tran); workToTranMap.put(work, tran); } @@ -129,16 +113,13 @@ private SparkTran generateParentTran(SparkPlan sparkPlan, SparkWork sparkWork, B if (work instanceof MapWork) { result = generateMapInput((MapWork)work); sparkPlan.addTran(result); - } else if (work instanceof ReduceWork) { + } else { List parentWorks = sparkWork.getParents(work); result = generate(sparkWork.getEdgeProperty(parentWorks.get(0), work), cloneToWork.containsKey(work)); sparkPlan.addTran(result); for (BaseWork parentWork : parentWorks) { sparkPlan.connect(workToTranMap.get(parentWork), result); } - } else { - throw new IllegalStateException("AssertionError: generateParentTran() only expect MapWork or ReduceWork," + - " but found " + work.getClass().getName()); } if (cloneToWork.containsKey(work)) { @@ -199,23 +180,21 @@ private ShuffleTran generate(SparkEdgeProperty edge, boolean toCache) { return new ShuffleTran(shuffler, edge.getNumPartitions(), toCache); } - private MapTran generate(MapWork mw) throws Exception { - initStatsPublisher(mw); - MapTran result = new MapTran(); - JobConf newJobConf = cloneJobConf(mw); + private SparkTran generate(BaseWork work) throws Exception { + initStatsPublisher(work); + JobConf newJobConf = cloneJobConf(work); byte[] confBytes = KryoSerializer.serializeJobConf(newJobConf); - HiveMapFunction mapFunc = new HiveMapFunction(confBytes, sparkReporter); - result.setMapFunction(mapFunc); - return result; - } - - private ReduceTran generate(ReduceWork rw) throws Exception { - ReduceTran result = new ReduceTran(); - JobConf newJobConf = cloneJobConf(rw); - byte[] confBytes = KryoSerializer.serializeJobConf(newJobConf); - HiveReduceFunction redFunc = new HiveReduceFunction(confBytes, sparkReporter); - result.setReduceFunction(redFunc); - return result; + if (work instanceof MapWork) { + MapTran mapTran = new MapTran(); + HiveMapFunction mapFunc = new HiveMapFunction(confBytes, sparkReporter); + mapTran.setMapFunction(mapFunc); + return mapTran; + } else { + ReduceTran reduceTran = new ReduceTran(); + HiveReduceFunction reduceFunc = new HiveReduceFunction(confBytes, sparkReporter); + reduceTran.setReduceFunction(reduceFunc); + return reduceTran; + } } private JobConf cloneJobConf(BaseWork work) throws Exception { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java index 78cbc6d..83625ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java @@ -150,16 +150,10 @@ public Object process(Node nd, Stack stack, * */ mapJoinWork = context.mapJoinWorkMap.get(mapJoinOp); - BaseWork parentWork; - if (context.unionWorkMap.containsKey(parentRS)) { - parentWork = context.unionWorkMap.get(parentRS); - } else { - int workMapSize = context.childToWorkMap.get(parentRS).size(); - Preconditions.checkArgument(workMapSize == 1, - "AssertionError: expected context.childToWorkMap.get(parentRS).size() to be 1, but was " + - workMapSize); - parentWork = context.childToWorkMap.get(parentRS).get(0); - } + int workMapSize = context.childToWorkMap.get(parentRS).size(); + Preconditions.checkArgument(workMapSize == 1, + "AssertionError: expected context.childToWorkMap.get(parentRS).size() to be 1, but was " + workMapSize); + BaseWork parentWork = context.childToWorkMap.get(parentRS).get(0); // set the link between mapjoin and parent vertex int pos = context.mapJoinParentMap.get(mapJoinOp).indexOf(parentRS); @@ -204,7 +198,6 @@ public Object process(Node nd, Stack stack, } // remember the output name of the reduce sink r.getConf().setOutputName(myWork.getName()); - context.connectedReduceSinks.add(r); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index ad6b09b..a4dfa6f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse.spark; +import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.DependencyCollectionTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -25,7 +26,6 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.UnionOperator; @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SparkEdgeProperty; import org.apache.hadoop.hive.ql.plan.SparkWork; @@ -86,9 +87,9 @@ // one. public BaseWork preceedingWork; - // map that keeps track of the last operator of a task to the work + // map that keeps track of the last operator of a task to the child work // that follows it. This is used for connecting them later. - public final Map, BaseWork> leafOperatorToFollowingWork; + public final Map> leafOpToChildWorkInfo; // a map that keeps track of work that need to be linked while // traversing an operator tree @@ -153,7 +154,8 @@ public GenSparkProcContext(HiveConf conf, ParseContext parseContext, this.currentTask = (SparkTask) TaskFactory.get( new SparkWork(conf.getVar(HiveConf.ConfVars.HIVEQUERYID)), conf); this.rootTasks.add(currentTask); - this.leafOperatorToFollowingWork = new LinkedHashMap, BaseWork>(); + this.leafOpToChildWorkInfo = + new LinkedHashMap>(); this.linkOpWithWorkMap = new LinkedHashMap, Map>(); this.linkWorkWithReduceSinkMap = new LinkedHashMap>(); this.smbJoinWorkMap = new LinkedHashMap(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 654ba33..f498e9e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -89,13 +89,6 @@ public void resetSequenceNumber() { sequenceNumber = 0; } - public UnionWork createUnionWork(GenSparkProcContext context, Operator operator, SparkWork sparkWork) { - UnionWork unionWork = new UnionWork("Union "+ (++sequenceNumber)); - context.unionWorkMap.put(operator, unionWork); - sparkWork.add(unionWork); - return unionWork; - } - public ReduceWork createReduceWork(GenSparkProcContext context, Operator root, SparkWork sparkWork) throws SemanticException { Preconditions.checkArgument(!root.getParentOperators().isEmpty(), "AssertionError: expected root.getParentOperators() to be non-empty"); @@ -122,10 +115,7 @@ public ReduceWork createReduceWork(GenSparkProcContext context, Operator root SparkEdgeProperty edgeProp = getEdgeProperty(reduceSink, reduceWork); - sparkWork.connect( - context.preceedingWork, - reduceWork, edgeProp); - context.connectedReduceSinks.add(reduceSink); + sparkWork.connect(context.preceedingWork, reduceWork, edgeProp); return reduceWork; } @@ -220,7 +210,7 @@ public void removeUnionOperators(Configuration conf, GenSparkProcContext context for (Operator op : opQueue) { Operator newOp = newOpQueue_it.next(); if (op instanceof FileSinkOperator) { - List fileSinkList = context.fileSinkMap.get((FileSinkOperator)op); + List fileSinkList = context.fileSinkMap.get(op); if (fileSinkList == null) { fileSinkList = new LinkedList(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java index 137df65..22bba99 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -36,11 +36,11 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SparkEdgeProperty; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.UnionWork; import java.util.ArrayList; import java.util.LinkedList; @@ -83,9 +83,8 @@ public Object process(Node nd, Stack stack, Preconditions.checkArgument(context.currentRootOperator != null, "AssertionError: expected context.currentRootOperator to be not null"); - // Operator is a file sink or reduce sink. Something that forces - // a new vertex. - Operator operator = (Operator) nd; + // Operator is a file sink or reduce sink. Something that forces a new vertex. + Operator operator = (Operator) nd; // root is the start of the operator pipeline we're currently // packing into a vertex, typically a table scan, union or join @@ -94,15 +93,8 @@ public Object process(Node nd, Stack stack, LOG.debug("Root operator: " + root); LOG.debug("Leaf operator: " + operator); - if (context.clonedReduceSinks.contains(operator)) { - // if we're visiting a terminal we've created ourselves, - // just skip and keep going - return null; - } - SparkWork sparkWork = context.currentTask.getWork(); - if (GenSparkUtils.getChildOperator(root, DummyStoreOperator.class) != null) { /* * SMB join case: @@ -120,7 +112,7 @@ public Object process(Node nd, Stack stack, */ return null; } - SMBMapJoinOperator smbOp = (SMBMapJoinOperator) GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class); + SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class); // Right now the work graph is pretty simple. If there is no // Preceding work we have a root and will generate a map @@ -140,9 +132,9 @@ public Object process(Node nd, Stack stack, // create a new vertex if (context.preceedingWork == null) { if (smbOp != null) { - //This logic is for SortMergeBucket MapJoin case. - //This MapWork (of big-table, see above..) is later initialized by SparkMapJoinFactory processor, so don't initialize it here. - //Just keep track of it in the context, for later processing. + // This logic is for SortMergeBucket MapJoin case. + // This MapWork (of big-table, see above..) is later initialized by SparkMapJoinFactory + // processor, so don't initialize it here. Just keep track of it in the context, for later processing. work = utils.createMapWork(context, root, sparkWork, null, true); if (context.smbJoinWorkMap.get(smbOp) != null) { throw new SemanticException("Each SMBMapJoin should be associated only with one Mapwork"); @@ -169,8 +161,7 @@ public Object process(Node nd, Stack stack, if (!context.currentMapJoinOperators.isEmpty()) { for (MapJoinOperator mj: context.currentMapJoinOperators) { LOG.debug("Processing map join: " + mj); - // remember the mapping in case we scan another branch of the - // mapjoin later + // remember the mapping in case we scan another branch of the mapjoin later if (!context.mapJoinWorkMap.containsKey(mj)) { List workItems = new LinkedList(); workItems.add(work); @@ -211,8 +202,7 @@ public Object process(Node nd, Stack stack, // need to set up output name for reduce sink now that we know the name // of the downstream work - for (ReduceSinkOperator r: - context.linkWorkWithReduceSinkMap.get(parentWork)) { + for (ReduceSinkOperator r : context.linkWorkWithReduceSinkMap.get(parentWork)) { if (r.getConf().getOutputName() != null) { LOG.debug("Cloning reduce sink for multi-child broadcast edge"); // we've already set this one up. Need to clone for the next work. @@ -221,7 +211,6 @@ public Object process(Node nd, Stack stack, context.clonedReduceSinks.add(r); } r.getConf().setOutputName(work.getName()); - context.connectedReduceSinks.add(r); } } } @@ -231,42 +220,32 @@ public Object process(Node nd, Stack stack, context.currentMapJoinOperators.clear(); } - // This is where we cut the tree as described above. We also remember that - // we might have to connect parent work with this work later. - for (Operator parent: new ArrayList>(root.getParentOperators())) { - context.leafOperatorToFollowingWork.put(parent, work); - LOG.debug("Removing " + parent + " as parent from " + root); - root.removeParent(parent); + // Here we are disconnecting root with its parents. However, we need to save + // a few information, since in future we may reach the parent operators via a + // different path, and we may need to connect parent works with the work associated + // with this root op. + if (root.getNumParent() > 0) { + Preconditions.checkArgument(work instanceof ReduceWork, + "AssertionError: expected work to be a ReduceWork, but was " + work.getClass().getName()); + ReduceWork reduceWork = (ReduceWork) work; + for (Operator parent : new ArrayList>(root.getParentOperators())) { + Preconditions.checkArgument(parent instanceof ReduceSinkOperator, + "AssertionError: expected operator to be a ReduceSinkOperator, but was " + parent.getClass().getName()); + ReduceSinkOperator rsOp = (ReduceSinkOperator) parent; + SparkEdgeProperty edgeProp = GenSparkUtils.getEdgeProperty(rsOp, reduceWork); + + rsOp.getConf().setOutputName(reduceWork.getName()); + GenMapRedUtils.setKeyAndValueDesc(reduceWork, rsOp); + + context.leafOpToChildWorkInfo.put(rsOp, ObjectPair.create(edgeProp, reduceWork)); + LOG.debug("Removing " + parent + " as parent from " + root); + root.removeParent(parent); + } } if (!context.currentUnionOperators.isEmpty()) { - // if there are union all operators we need to add the work to the set - // of union operators. - - UnionWork unionWork; - if (context.unionWorkMap.containsKey(operator)) { - // we've seen this terminal before and have created a union work object. - // just need to add this work to it. There will be no children of this one - // since we've passed this operator before. - Preconditions.checkArgument(operator.getChildOperators().isEmpty(), - "AssertionError: expected operator.getChildOperators() to be empty"); - unionWork = (UnionWork) context.unionWorkMap.get(operator); - - } else { - // first time through. we need to create a union work object and add this - // work to it. Subsequent work should reference the union and not the actual - // work. - unionWork = utils.createUnionWork(context, operator, sparkWork); - } - - // finally hook everything up - LOG.debug("Connecting union work ("+unionWork+") with work ("+work+")"); - SparkEdgeProperty edgeProp = new SparkEdgeProperty(SparkEdgeProperty.SHUFFLE_NONE); - sparkWork.connect(work, unionWork, edgeProp); - unionWork.addUnionOperators(context.currentUnionOperators); context.currentUnionOperators.clear(); context.workWithUnionOperators.add(work); - work = unionWork; } // We're scanning a tree from roots to leaf (this is not technically @@ -280,39 +259,36 @@ public Object process(Node nd, Stack stack, // // Also note: the concept of leaf and root is reversed in hive for historical // reasons. Roots are data sources, leaves are data sinks. I know. - if (context.leafOperatorToFollowingWork.containsKey(operator)) { - - BaseWork followingWork = context.leafOperatorToFollowingWork.get(operator); - long bytesPerReducer = context.conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER); - - LOG.debug("Second pass. Leaf operator: "+operator - +" has common downstream work:"+followingWork); - - // need to add this branch to the key + value info - Preconditions.checkArgument(operator instanceof ReduceSinkOperator, - "AssertionError: expected operator to be an instance of ReduceSinkOperator, but was " + - operator.getClass().getName()); - Preconditions.checkArgument(followingWork instanceof ReduceWork, - "AssertionError: expected followingWork to be an instance of ReduceWork, but was " + - followingWork.getClass().getName()); - ReduceSinkOperator rs = (ReduceSinkOperator) operator; - ReduceWork rWork = (ReduceWork) followingWork; - GenMapRedUtils.setKeyAndValueDesc(rWork, rs); - - // remember which parent belongs to which tag - rWork.getTagToInput().put(rs.getConf().getTag(), work.getName()); - - // remember the output name of the reduce sink - rs.getConf().setOutputName(rWork.getName()); - - if (!context.connectedReduceSinks.contains(rs)) { - // add dependency between the two work items - SparkEdgeProperty edgeProp = GenSparkUtils.getEdgeProperty(rs, rWork); - sparkWork.connect(work, rWork, edgeProp); - context.connectedReduceSinks.add(rs); + if (context.leafOpToChildWorkInfo.containsKey(operator)) { + ObjectPair followingWorkInfo = context.leafOpToChildWorkInfo.get(operator); + SparkEdgeProperty edgeProp = followingWorkInfo.getFirst(); + ReduceWork followingWork = followingWorkInfo.getSecond(); + + LOG.debug("Second pass. Leaf operator: " + operator + " has common downstream work:" + followingWork); + + // We may have already connected `work` with `followingWork`, in case, for example, lateral view: + // TS + // | + // ... + // | + // LVF + // | \ + // SEL SEL + // | | + // LVJ-UDTF + // | + // SEL + // | + // RS + // Here, RS can be reached from TS via two different paths. If there is any following work after RS, + // we don't want to connect them with the work associated with TS more than once. + if (sparkWork.getEdgeProperty(work, followingWork) == null) { + sparkWork.connect(work, followingWork, edgeProp); + } else { + LOG.debug("work " + work.getName() + " is already connected to " + followingWork.getName() + " before"); } } else { - LOG.debug("First pass. Leaf operator: "+operator); + LOG.debug("First pass. Leaf operator: " + operator); } // No children means we're at the bottom. If there are more operators to scan diff --git ql/src/test/results/clientpositive/spark/auto_join27.q.out ql/src/test/results/clientpositive/spark/auto_join27.q.out index fb48351..ca389d4 100644 --- ql/src/test/results/clientpositive/spark/auto_join27.q.out +++ ql/src/test/results/clientpositive/spark/auto_join27.q.out @@ -30,10 +30,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -49,7 +48,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: src @@ -67,7 +66,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: src @@ -84,7 +83,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Reducer 2 Reduce Operator Tree: Join Operator condition map: @@ -102,7 +101,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -120,7 +119,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) @@ -133,8 +132,6 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out index 8472df9..ca7bc32 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out @@ -72,9 +72,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3) - Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -93,7 +92,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Map 5 + Map 4 Map Operator Tree: TableScan alias: a @@ -109,7 +108,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Map 6 + Map 5 Map Operator Tree: TableScan alias: a @@ -126,7 +125,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Reducer 2 Reduce Operator Tree: Join Operator condition map: @@ -144,7 +143,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -162,8 +161,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/column_access_stats.q.out ql/src/test/results/clientpositive/spark/column_access_stats.q.out index 72b2bd7..68ac466 100644 --- ql/src/test/results/clientpositive/spark/column_access_stats.q.out +++ ql/src/test/results/clientpositive/spark/column_access_stats.q.out @@ -185,8 +185,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -205,7 +203,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -221,8 +219,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -267,8 +263,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -287,7 +281,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -303,8 +297,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 1757d16..995b69f 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -1948,8 +1948,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -2051,7 +2049,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -2150,8 +2148,6 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -2313,8 +2309,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 3) - Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2419,7 +2414,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -2496,7 +2491,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Reducer 4 + Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -2537,8 +2532,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 04f481d..4a38ca6 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -2020,8 +2020,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -2123,7 +2121,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -2222,8 +2220,6 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -2385,9 +2381,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3) - Reducer 5 <- Reducer 4 (GROUP, 3) - Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2492,7 +2487,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Map 3 + Map 2 Map Operator Tree: TableScan alias: t1 @@ -2569,7 +2564,7 @@ STAGE PLANS: name: default.t1 Truncated Path -> Alias: /t1 [t1] - Reducer 4 + Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -2586,7 +2581,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false - Reducer 5 + Reducer 4 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -2627,8 +2622,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/join34.q.out ql/src/test/results/clientpositive/spark/join34.q.out index 9a58a22..f6882b4 100644 --- ql/src/test/results/clientpositive/spark/join34.q.out +++ ql/src/test/results/clientpositive/spark/join34.q.out @@ -147,8 +147,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -218,7 +217,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [x] - Map 4 + Map 3 Map Operator Tree: TableScan alias: x1 @@ -285,7 +284,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [x1] - Map 5 + Map 4 Map Operator Tree: TableScan alias: x @@ -352,7 +351,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 3 + Reducer 2 Needs Tagging: true Reduce Operator Tree: Join Operator @@ -393,8 +392,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/join35.q.out ql/src/test/results/clientpositive/spark/join35.q.out index 851a981..8e9aed8 100644 --- ql/src/test/results/clientpositive/spark/join35.q.out +++ ql/src/test/results/clientpositive/spark/join35.q.out @@ -156,9 +156,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -234,7 +233,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [x] - Map 5 + Map 4 Map Operator Tree: TableScan alias: x1 @@ -307,7 +306,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [x1] - Map 7 + Map 6 Map Operator Tree: TableScan alias: x @@ -389,7 +388,7 @@ STAGE PLANS: tag: 0 value expressions: _col1 (type: bigint) auto parallelism: false - Reducer 4 + Reducer 3 Needs Tagging: true Reduce Operator Tree: Join Operator @@ -430,7 +429,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Reducer 6 + Reducer 5 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -445,8 +444,6 @@ STAGE PLANS: tag: 0 value expressions: _col1 (type: bigint) auto parallelism: false - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out index 92693e6..51e57e8 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out @@ -64,8 +64,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -84,7 +82,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part13 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -100,8 +98,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part13 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out index 060745d..1f9985f 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out @@ -63,9 +63,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 7 <- Map 6 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0), Reducer 7 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) + Reducer 6 <- Map 5 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -84,7 +83,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: src @@ -100,7 +99,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: src @@ -130,7 +129,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part14 - Reducer 5 + Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) @@ -144,7 +143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part14 - Reducer 7 + Reducer 6 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) @@ -158,8 +157,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part14 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/multi_insert.q.out ql/src/test/results/clientpositive/spark/multi_insert.q.out index 0a38bea..ae4899f 100644 --- ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -1170,8 +1170,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1199,7 +1197,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1224,8 +1222,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -1345,8 +1341,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1374,7 +1368,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1399,8 +1393,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -1520,8 +1512,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1549,7 +1539,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1574,8 +1564,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -1695,8 +1683,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1724,7 +1710,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1749,8 +1735,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 639f4bd..c2056e0 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -1203,8 +1203,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1232,7 +1230,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1257,8 +1255,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1382,8 +1378,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1411,7 +1405,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1436,8 +1430,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1561,8 +1553,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1590,7 +1580,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1615,8 +1605,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1740,8 +1728,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -1769,7 +1755,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1794,8 +1780,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_multi2 - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection diff --git ql/src/test/results/clientpositive/spark/multi_join_union.q.out ql/src/test/results/clientpositive/spark/multi_join_union.q.out index d8dc110..d6f1374 100644 --- ql/src/test/results/clientpositive/spark/multi_join_union.q.out +++ ql/src/test/results/clientpositive/spark/multi_join_union.q.out @@ -62,7 +62,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -83,8 +83,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3) - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -101,7 +100,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) value expressions: _col0 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: src14 @@ -115,7 +114,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) value expressions: _col0 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: a @@ -134,7 +133,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6 input vertices: - 1 Map 6 + 1 Map 5 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col6 (type: string) @@ -144,7 +143,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Reducer 3 + Reducer 2 Reduce Operator Tree: Join Operator condition map: @@ -165,8 +164,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out index 029cd33..9a8ff4e 100644 --- ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out +++ ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out @@ -650,8 +650,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -724,7 +723,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: -mr-10003default.src{} [src] - Map 4 + Map 3 Map Operator Tree: TableScan alias: srcpart @@ -967,7 +966,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Reducer 5 + Reducer 4 Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -997,8 +996,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1527,8 +1524,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1614,7 +1610,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -1680,8 +1676,8 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: -mr-10003default.src{} [src] - Map 5 - Reducer 4 + Map 4 + Reducer 3 Needs Tagging: true Reduce Operator Tree: Join Operator @@ -1714,8 +1710,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out index db92598..a356fd1 100644 --- ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out @@ -68,9 +68,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -87,7 +86,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -101,7 +100,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -115,7 +114,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -147,7 +146,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -165,8 +164,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -210,9 +207,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -229,7 +225,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -243,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -257,7 +253,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -289,7 +285,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -307,8 +303,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -360,9 +354,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -379,7 +372,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -393,7 +386,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -407,7 +400,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -440,7 +433,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -459,8 +452,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator @@ -520,9 +511,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -539,7 +529,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -553,7 +543,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -567,7 +557,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -600,7 +590,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -619,8 +609,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/skewjoin_union_remove_2.q.out ql/src/test/results/clientpositive/spark/skewjoin_union_remove_2.q.out index bbe60a7..84bbbf6 100644 --- ql/src/test/results/clientpositive/spark/skewjoin_union_remove_2.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin_union_remove_2.q.out @@ -80,9 +80,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1), Map 9 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -99,7 +98,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -113,7 +112,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: c @@ -127,7 +126,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -141,7 +140,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 8 + Map 7 Map Operator Tree: TableScan alias: c @@ -155,7 +154,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 9 + Map 8 Map Operator Tree: TableScan alias: a @@ -189,7 +188,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -209,8 +208,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out index c3d550b..8560650 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out @@ -54,9 +54,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -73,7 +72,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -87,7 +86,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -101,7 +100,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -133,7 +132,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -151,8 +150,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -196,9 +193,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -215,7 +211,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -229,7 +225,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -243,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -275,7 +271,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -293,8 +289,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -340,10 +334,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 6 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -359,7 +352,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -372,7 +365,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -385,7 +378,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -413,7 +406,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -431,7 +424,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -446,8 +439,6 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -480,10 +471,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 6 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -499,7 +489,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -512,7 +502,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -525,7 +515,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -553,7 +543,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -571,7 +561,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -586,8 +576,6 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out index 4bb908a..f559549 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out @@ -56,9 +56,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -74,7 +73,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -88,7 +87,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: array) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -102,7 +101,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: array) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -159,7 +158,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -203,8 +202,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt11.q.out ql/src/test/results/clientpositive/spark/skewjoinopt11.q.out index bb1111a..e86128d 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt11.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt11.q.out @@ -66,11 +66,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 1), Map 13 (PARTITION-LEVEL SORT, 1) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Reducer 9 <- Map 10 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 12 (NONE, 0), Reducer 2 (NONE, 0), Reducer 6 (NONE, 0), Reducer 9 (NONE, 0) + Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 1), Map 12 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) + Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 1), Map 9 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -90,7 +89,7 @@ STAGE PLANS: Map 10 Map Operator Tree: TableScan - alias: a + alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key is not null and (not (key = '2'))) (type: boolean) @@ -101,10 +100,10 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 11 + Map 12 Map Operator Tree: TableScan - alias: b + alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key is not null and (not (key = '2'))) (type: boolean) @@ -115,13 +114,13 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 13 + Map 3 Map Operator Tree: TableScan - alias: a + alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key is not null and (not (key = '2'))) (type: boolean) + predicate: (key is not null and (key = '2')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -132,7 +131,7 @@ STAGE PLANS: Map 4 Map Operator Tree: TableScan - alias: b + alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key is not null and (key = '2')) (type: boolean) @@ -143,10 +142,10 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 6 Map Operator Tree: TableScan - alias: a + alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key is not null and (key = '2')) (type: boolean) @@ -163,7 +162,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (key is not null and (key = '2')) (type: boolean) + predicate: (key is not null and (not (key = '2'))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: key (type: string) @@ -171,10 +170,10 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 8 + Map 9 Map Operator Tree: TableScan - alias: b + alias: a Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key is not null and (not (key = '2'))) (type: boolean) @@ -185,7 +184,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Reducer 12 + Reducer 11 Reduce Operator Tree: Join Operator condition map: @@ -227,7 +226,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -248,7 +247,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Reducer 8 Reduce Operator Tree: Join Operator condition map: @@ -269,8 +268,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt12.q.out ql/src/test/results/clientpositive/spark/skewjoinopt12.q.out index 468139c..b4deb0f 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt12.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt12.q.out @@ -56,9 +56,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -74,7 +73,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -87,7 +86,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -100,7 +99,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -131,7 +130,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -149,8 +148,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt14.q.out ql/src/test/results/clientpositive/spark/skewjoinopt14.q.out index b3dcb0a..f9004ed 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt14.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt14.q.out @@ -86,10 +86,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1) - Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 1), Map 9 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 8 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 7 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -106,7 +105,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -120,7 +119,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: c @@ -134,7 +133,7 @@ STAGE PLANS: Map-reduce partition columns: val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: key (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: b @@ -148,7 +147,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 9 + Map 8 Map Operator Tree: TableScan alias: a @@ -176,7 +175,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) value expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string) - Reducer 4 + Reducer 3 Reduce Operator Tree: Join Operator condition map: @@ -197,7 +196,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Reducer 7 Reduce Operator Tree: Join Operator condition map: @@ -211,8 +210,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) value expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out index e62e886..4d9b7dd 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out @@ -94,9 +94,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -113,7 +112,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -127,7 +126,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -141,7 +140,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -173,7 +172,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -191,8 +190,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -236,9 +233,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -255,7 +251,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -269,7 +265,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -283,7 +279,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -315,7 +311,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -333,8 +329,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -380,10 +374,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 6 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -399,7 +392,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -412,7 +405,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -425,7 +418,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -453,7 +446,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -471,7 +464,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -486,8 +479,6 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -520,10 +511,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 6 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -539,7 +529,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -552,7 +542,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -565,7 +555,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -593,7 +583,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -611,7 +601,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -626,8 +616,6 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt16.q.out ql/src/test/results/clientpositive/spark/skewjoinopt16.q.out index 771b16e..7bb4d83 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt16.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt16.q.out @@ -56,9 +56,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -74,7 +73,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -87,7 +86,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -100,7 +99,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -131,7 +130,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -149,8 +148,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt17.q.out ql/src/test/results/clientpositive/spark/skewjoinopt17.q.out index f59f1e4..8267ef3 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt17.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt17.q.out @@ -60,9 +60,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -79,7 +78,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -93,7 +92,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -107,7 +106,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -139,7 +138,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -157,8 +156,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -258,9 +255,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -276,7 +272,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -289,7 +285,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -302,7 +298,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -333,7 +329,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -351,8 +347,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt19.q.out ql/src/test/results/clientpositive/spark/skewjoinopt19.q.out index bac15f6..181ecae 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt19.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt19.q.out @@ -58,9 +58,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -77,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -91,7 +90,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -137,7 +136,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -155,8 +154,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out index f6a47de..845504c 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out @@ -64,9 +64,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -82,7 +81,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -95,7 +94,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -108,7 +107,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -139,7 +138,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -157,8 +156,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -199,9 +196,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -217,7 +213,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -230,7 +226,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -243,7 +239,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -274,7 +270,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -292,8 +288,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -338,10 +332,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3), Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -357,7 +350,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -370,7 +363,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -383,7 +376,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -415,7 +408,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -434,7 +427,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -453,8 +446,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -488,10 +479,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 3), Map 8 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3), Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -507,7 +497,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -520,7 +510,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -533,7 +523,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: key (type: string), val (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Map 8 + Map 7 Map Operator Tree: TableScan alias: a @@ -565,7 +555,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -584,7 +574,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -603,8 +593,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt20.q.out ql/src/test/results/clientpositive/spark/skewjoinopt20.q.out index 4150c23..827fa40 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt20.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt20.q.out @@ -58,9 +58,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -77,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -91,7 +90,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -137,7 +136,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -155,8 +154,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out index 5640384..8094603 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out @@ -58,9 +58,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -77,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -91,7 +90,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -137,7 +136,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -155,8 +154,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -200,9 +197,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -219,7 +215,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -233,7 +229,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -247,7 +243,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -279,7 +275,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -297,8 +293,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt4.q.out ql/src/test/results/clientpositive/spark/skewjoinopt4.q.out index 353e1b8..e4ab45f 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt4.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt4.q.out @@ -54,9 +54,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -73,7 +72,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -87,7 +86,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -101,7 +100,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -133,7 +132,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -151,8 +150,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -194,9 +191,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -213,7 +209,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -227,7 +223,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -241,7 +237,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -273,7 +269,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -291,8 +287,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt5.q.out ql/src/test/results/clientpositive/spark/skewjoinopt5.q.out index 4fe594c..a038ad0 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt5.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt5.q.out @@ -56,9 +56,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -75,7 +74,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -89,7 +88,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -103,7 +102,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -135,7 +134,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -153,8 +152,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt6.q.out ql/src/test/results/clientpositive/spark/skewjoinopt6.q.out index fc1e81c..1175729 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt6.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt6.q.out @@ -58,9 +58,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -77,7 +76,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -91,7 +90,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 7 + Map 6 Map Operator Tree: TableScan alias: a @@ -137,7 +136,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Join Operator condition map: @@ -155,8 +154,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt7.q.out ql/src/test/results/clientpositive/spark/skewjoinopt7.q.out index f76ebbd..5791aeb 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt7.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt7.q.out @@ -74,9 +74,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1), Map 9 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -93,7 +92,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -107,7 +106,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: c @@ -121,7 +120,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -135,7 +134,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 8 + Map 7 Map Operator Tree: TableScan alias: c @@ -149,7 +148,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 9 + Map 8 Map Operator Tree: TableScan alias: a @@ -183,7 +182,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -203,8 +202,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt8.q.out ql/src/test/results/clientpositive/spark/skewjoinopt8.q.out index 466d70e..b23bb5b 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt8.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt8.q.out @@ -72,9 +72,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1), Map 9 (PARTITION-LEVEL SORT, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 7 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -91,7 +90,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: b @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: c @@ -119,7 +118,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: b @@ -133,7 +132,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 8 + Map 7 Map Operator Tree: TableScan alias: c @@ -147,7 +146,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Map 9 + Map 8 Map Operator Tree: TableScan alias: a @@ -181,7 +180,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 6 Reduce Operator Tree: Join Operator condition map: @@ -201,8 +200,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out index bc0b1f7..51058e2 100644 --- ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out +++ ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out @@ -64,8 +64,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -82,7 +81,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: t1 @@ -96,7 +95,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: b @@ -110,7 +109,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: val (type: string) - Reducer 3 + Reducer 2 Reduce Operator Tree: Join Operator condition map: @@ -131,8 +130,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/stats1.q.out ql/src/test/results/clientpositive/spark/stats1.q.out index ba22d9a..a993a66 100644 --- ql/src/test/results/clientpositive/spark/stats1.q.out +++ ql/src/test/results/clientpositive/spark/stats1.q.out @@ -30,7 +30,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -49,7 +48,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -79,8 +78,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/temp_table.q.out ql/src/test/results/clientpositive/spark/temp_table.q.out index feb7711..16d663d 100644 --- ql/src/test/results/clientpositive/spark/temp_table.q.out +++ ql/src/test/results/clientpositive/spark/temp_table.q.out @@ -200,8 +200,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) - Reducer 3 <- Union 2 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 1), Map 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -215,7 +214,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + value expressions: _col1 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: bar @@ -226,7 +225,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + value expressions: _col1 (type: string) - Reducer 3 + Reducer 2 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) @@ -242,8 +241,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union.q.out ql/src/test/results/clientpositive/spark/union.q.out index 7d8d452..4c8a90f 100644 --- ql/src/test/results/clientpositive/spark/union.q.out +++ ql/src/test/results/clientpositive/spark/union.q.out @@ -27,8 +27,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -49,7 +47,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -67,8 +65,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union10.q.out ql/src/test/results/clientpositive/spark/union10.q.out index 40a43c6..74673fb 100644 --- ql/src/test/results/clientpositive/spark/union10.q.out +++ ql/src/test/results/clientpositive/spark/union10.q.out @@ -36,9 +36,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 7 <- Map 6 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0), Reducer 7 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) + Reducer 6 <- Map 5 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -57,7 +56,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -73,7 +72,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Map 5 Map Operator Tree: TableScan alias: s3 @@ -108,7 +107,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -127,7 +126,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 7 + Reducer 6 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -146,8 +145,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union11.q.out ql/src/test/results/clientpositive/spark/union11.q.out index 068f7dc..74cca13 100644 --- ql/src/test/results/clientpositive/spark/union11.q.out +++ ql/src/test/results/clientpositive/spark/union11.q.out @@ -27,10 +27,9 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Union 3 (GROUP, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Reducer 8 <- Map 7 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0), Reducer 8 (NONE, 0) + Reducer 5 <- Map 4 (GROUP, 1) + Reducer 7 <- Map 6 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 5 (GROUP, 1), Reducer 7 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -49,7 +48,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s2 @@ -65,7 +64,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 7 + Map 6 Map Operator Tree: TableScan alias: s3 @@ -103,7 +102,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -122,7 +121,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -144,7 +143,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 8 + Reducer 7 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -166,8 +165,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union13.q.out ql/src/test/results/clientpositive/spark/union13.q.out index 27de88b..d0316f2 100644 --- ql/src/test/results/clientpositive/spark/union13.q.out +++ ql/src/test/results/clientpositive/spark/union13.q.out @@ -19,8 +19,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -39,7 +37,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + Map 2 Map Operator Tree: TableScan alias: s2 @@ -55,8 +53,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union14.q.out ql/src/test/results/clientpositive/spark/union14.q.out index 47f4ac1..526a907 100644 --- ql/src/test/results/clientpositive/spark/union14.q.out +++ ql/src/test/results/clientpositive/spark/union14.q.out @@ -24,9 +24,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Union 2 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 1), Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -49,7 +48,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s1 @@ -65,7 +64,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -84,7 +83,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -106,8 +105,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union15.q.out ql/src/test/results/clientpositive/spark/union15.q.out index 487fe53..a450fd6 100644 --- ql/src/test/results/clientpositive/spark/union15.q.out +++ ql/src/test/results/clientpositive/spark/union15.q.out @@ -27,8 +27,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Union 3 (GROUP, 1) - Union 3 <- Map 5 (NONE, 0), Map 6 (NONE, 0), Reducer 2 (NONE, 0) + Reducer 3 <- Map 4 (GROUP, 1), Map 5 (GROUP, 1), Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -47,7 +46,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s2 @@ -67,7 +66,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Map 6 + Map 5 Map Operator Tree: TableScan alias: s3 @@ -109,7 +108,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -128,8 +127,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union16.q.out ql/src/test/results/clientpositive/spark/union16.q.out index c35ed10..f7025f7 100644 --- ql/src/test/results/clientpositive/spark/union16.q.out +++ ql/src/test/results/clientpositive/spark/union16.q.out @@ -72,8 +72,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Union 2 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Map 10 (NONE, 0), Map 11 (NONE, 0), Map 12 (NONE, 0), Map 13 (NONE, 0), Map 14 (NONE, 0), Map 15 (NONE, 0), Map 16 (NONE, 0), Map 17 (NONE, 0), Map 18 (NONE, 0), Map 19 (NONE, 0), Map 20 (NONE, 0), Map 21 (NONE, 0), Map 22 (NONE, 0), Map 23 (NONE, 0), Map 24 (NONE, 0), Map 25 (NONE, 0), Map 26 (NONE, 0), Map 27 (NONE, 0), Map 4 (NONE, 0), Map 5 (NONE, 0), Map 6 (NONE, 0), Map 7 (NONE, 0), Map 8 (NONE, 0), Map 9 (NONE, 0) + Reducer 2 <- Map 1 (GROUP, 1), Map 10 (GROUP, 1), Map 11 (GROUP, 1), Map 12 (GROUP, 1), Map 13 (GROUP, 1), Map 14 (GROUP, 1), Map 15 (GROUP, 1), Map 16 (GROUP, 1), Map 17 (GROUP, 1), Map 18 (GROUP, 1), Map 19 (GROUP, 1), Map 20 (GROUP, 1), Map 21 (GROUP, 1), Map 22 (GROUP, 1), Map 23 (GROUP, 1), Map 24 (GROUP, 1), Map 25 (GROUP, 1), Map 26 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1), Map 5 (GROUP, 1), Map 6 (GROUP, 1), Map 7 (GROUP, 1), Map 8 (GROUP, 1), Map 9 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -310,7 +309,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Map 27 + Map 3 Map Operator Tree: TableScan alias: src @@ -401,7 +400,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -419,8 +418,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union18.q.out ql/src/test/results/clientpositive/spark/union18.q.out index d2bcd70..e751d2c 100644 --- ql/src/test/results/clientpositive/spark/union18.q.out +++ ql/src/test/results/clientpositive/spark/union18.q.out @@ -44,7 +44,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -63,7 +62,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -113,8 +112,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union19.q.out ql/src/test/results/clientpositive/spark/union19.q.out index 13fb395..ed38ce9 100644 --- ql/src/test/results/clientpositive/spark/union19.q.out +++ ql/src/test/results/clientpositive/spark/union19.q.out @@ -44,8 +44,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 5 (NONE, 0), Reducer 2 (NONE, 0) - Reducer 4 <- Union 3 (GROUP, 1) + Reducer 3 <- Map 4 (GROUP, 1), Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -64,7 +63,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s2 @@ -120,7 +119,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -136,8 +135,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union2.q.out ql/src/test/results/clientpositive/spark/union2.q.out index da8d154..4a99714 100644 --- ql/src/test/results/clientpositive/spark/union2.q.out +++ ql/src/test/results/clientpositive/spark/union2.q.out @@ -20,8 +20,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Union 2 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) + Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -37,7 +36,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -50,7 +49,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -68,8 +67,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union23.q.out ql/src/test/results/clientpositive/spark/union23.q.out index 606153a..9bc6bd0 100644 --- ql/src/test/results/clientpositive/spark/union23.q.out +++ ql/src/test/results/clientpositive/spark/union23.q.out @@ -24,8 +24,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) - Reducer 3 <- Union 2 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 1), Map 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -47,7 +46,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map 4 + Map 3 Map Operator Tree: TableScan alias: src @@ -60,7 +59,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Reducer 3 + Reducer 2 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) @@ -73,8 +72,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union25.q.out ql/src/test/results/clientpositive/spark/union25.q.out index c439b1a..854638b 100644 --- ql/src/test/results/clientpositive/spark/union25.q.out +++ ql/src/test/results/clientpositive/spark/union25.q.out @@ -66,10 +66,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Union 5 <- Map 4 (NONE, 0), Map 7 (NONE, 0) - Reducer 6 <- Union 5 (GROUP, 3) - Union 2 <- Map 1 (NONE, 0), Reducer 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 3) + Reducer 4 <- Map 3 (GROUP, 3), Map 5 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 3), Reducer 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -89,7 +87,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) value expressions: _col2 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: a @@ -107,7 +105,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Map 7 + Map 5 Map Operator Tree: TableScan alias: b @@ -125,7 +123,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -145,7 +143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmp_unionall - Reducer 6 + Reducer 4 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -164,10 +162,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) value expressions: _col2 (type: bigint) - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union28.q.out ql/src/test/results/clientpositive/spark/union28.q.out index b478a77..aac2962 100644 --- ql/src/test/results/clientpositive/spark/union28.q.out +++ ql/src/test/results/clientpositive/spark/union28.q.out @@ -41,9 +41,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 3 <- Map 2 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -63,7 +62,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -84,7 +83,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: src @@ -105,7 +104,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -128,7 +127,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -151,8 +150,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union29.q.out ql/src/test/results/clientpositive/spark/union29.q.out index da22456..daa581f 100644 --- ql/src/test/results/clientpositive/spark/union29.q.out +++ ql/src/test/results/clientpositive/spark/union29.q.out @@ -40,8 +40,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0), Map 4 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -61,7 +59,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -78,7 +76,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Map 4 + Map 3 Map Operator Tree: TableScan alias: src @@ -95,8 +93,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union3.q.out ql/src/test/results/clientpositive/spark/union3.q.out index 8240654..788bddd 100644 --- ql/src/test/results/clientpositive/spark/union3.q.out +++ ql/src/test/results/clientpositive/spark/union3.q.out @@ -46,13 +46,12 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 11 <- Map 10 (GROUP, 1) + Reducer 10 <- Map 9 (GROUP, 1) Reducer 2 <- Map 1 (GROUP, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Reducer 9 <- Map 8 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 1) + Reducer 8 <- Map 7 (GROUP, 1) Reducer 3 <- Reducer 2 (SORT, 1) - Reducer 7 <- Reducer 6 (SORT, 1) - Union 4 <- Reducer 11 (NONE, 0), Reducer 3 (NONE, 0), Reducer 7 (NONE, 0), Reducer 9 (NONE, 0) + Reducer 6 <- Reducer 5 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -68,7 +67,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 10 + Map 4 Map Operator Tree: TableScan alias: src @@ -81,7 +80,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 5 + Map 7 Map Operator Tree: TableScan alias: src @@ -94,7 +93,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Map 8 + Map 9 Map Operator Tree: TableScan alias: src @@ -107,7 +106,7 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 11 + Reducer 10 Reduce Operator Tree: Limit Number of rows: 1 @@ -151,7 +150,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Limit Number of rows: 1 @@ -165,7 +164,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reducer 7 + Reducer 6 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int) @@ -179,7 +178,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Reducer 8 Reduce Operator Tree: Limit Number of rows: 1 @@ -195,8 +194,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union30.q.out ql/src/test/results/clientpositive/spark/union30.q.out index c4eeb8d..df94a96 100644 --- ql/src/test/results/clientpositive/spark/union30.q.out +++ ql/src/test/results/clientpositive/spark/union30.q.out @@ -55,9 +55,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Map 7 (NONE, 0), Reducer 4 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 3 <- Map 2 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -77,7 +76,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -98,7 +97,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: src @@ -119,7 +118,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Map 7 + Map 6 Map Operator Tree: TableScan alias: src @@ -136,7 +135,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -159,7 +158,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -182,8 +181,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union33.q.out ql/src/test/results/clientpositive/spark/union33.q.out index b5a3f99..7c6ac28 100644 --- ql/src/test/results/clientpositive/spark/union33.q.out +++ ql/src/test/results/clientpositive/spark/union33.q.out @@ -41,9 +41,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -62,7 +61,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src - Map 3 + Map 2 Map Operator Tree: TableScan alias: src @@ -83,7 +82,7 @@ STAGE PLANS: Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -97,7 +96,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -114,8 +113,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator @@ -191,7 +188,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) - Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0) Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: @@ -216,7 +212,7 @@ STAGE PLANS: Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: src @@ -263,8 +259,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.test_src - Union 4 - Vertex: Union 4 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union4.q.out ql/src/test/results/clientpositive/spark/union4.q.out index 255df76..5f2e76a 100644 --- ql/src/test/results/clientpositive/spark/union4.q.out +++ ql/src/test/results/clientpositive/spark/union4.q.out @@ -34,8 +34,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -54,7 +53,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -89,7 +88,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -108,8 +107,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union5.q.out ql/src/test/results/clientpositive/spark/union5.q.out index 2d8a3c4..bb81548 100644 --- ql/src/test/results/clientpositive/spark/union5.q.out +++ ql/src/test/results/clientpositive/spark/union5.q.out @@ -23,9 +23,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Union 3 (GROUP, 1) - Reducer 6 <- Map 5 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) + Reducer 5 <- Map 4 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 1), Reducer 5 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -44,7 +43,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s2 @@ -82,7 +81,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -101,7 +100,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -123,8 +122,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union6.q.out ql/src/test/results/clientpositive/spark/union6.q.out index eb1c751..6d523eb 100644 --- ql/src/test/results/clientpositive/spark/union6.q.out +++ ql/src/test/results/clientpositive/spark/union6.q.out @@ -34,7 +34,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -53,7 +52,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -83,8 +82,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union7.q.out ql/src/test/results/clientpositive/spark/union7.q.out index 5fb37da..948c86c 100644 --- ql/src/test/results/clientpositive/spark/union7.q.out +++ ql/src/test/results/clientpositive/spark/union7.q.out @@ -23,8 +23,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Union 3 (GROUP, 1) - Union 3 <- Map 5 (NONE, 0), Reducer 2 (NONE, 0) + Reducer 3 <- Map 4 (GROUP, 1), Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -43,7 +42,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s2 @@ -85,7 +84,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) value expressions: _col1 (type: bigint) - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -104,8 +103,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union8.q.out ql/src/test/results/clientpositive/spark/union8.q.out index 4e5cec5..1962937 100644 --- ql/src/test/results/clientpositive/spark/union8.q.out +++ ql/src/test/results/clientpositive/spark/union8.q.out @@ -21,8 +21,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0), Map 4 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -41,7 +39,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + Map 2 Map Operator Tree: TableScan alias: s2 @@ -57,7 +55,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 4 + Map 3 Map Operator Tree: TableScan alias: s3 @@ -73,8 +71,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union9.q.out ql/src/test/results/clientpositive/spark/union9.q.out index db14477..4b3fafa 100644 --- ql/src/test/results/clientpositive/spark/union9.q.out +++ ql/src/test/results/clientpositive/spark/union9.q.out @@ -22,8 +22,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Union 2 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0), Map 5 (NONE, 0) + Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -39,7 +38,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: s2 @@ -52,7 +51,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: s3 @@ -65,7 +64,7 @@ STAGE PLANS: Reduce Output Operator sort order: value expressions: _col0 (type: bigint) - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -83,8 +82,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union_ppr.q.out ql/src/test/results/clientpositive/spark/union_ppr.q.out index b907953..611ad23 100644 --- ql/src/test/results/clientpositive/spark/union_ppr.q.out +++ ql/src/test/results/clientpositive/spark/union_ppr.q.out @@ -112,8 +112,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Union 2 <- Map 1 (NONE, 0), Map 4 (NONE, 0) - Reducer 3 <- Union 2 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 1), Map 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -233,7 +232,7 @@ STAGE PLANS: Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [x] /srcpart/ds=2008-04-08/hr=12 [x] - Map 4 + Map 3 Map Operator Tree: TableScan alias: y @@ -350,7 +349,7 @@ STAGE PLANS: Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [y] /srcpart/ds=2008-04-08/hr=12 [y] - Reducer 3 + Reducer 2 Needs Tagging: false Reduce Operator Tree: Select Operator @@ -378,8 +377,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_1.q.out ql/src/test/results/clientpositive/spark/union_remove_1.q.out index be6e0e9..f2095ee 100644 --- ql/src/test/results/clientpositive/spark/union_remove_1.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_1.q.out @@ -69,8 +69,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -94,7 +93,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -129,7 +128,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -143,8 +142,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_10.q.out ql/src/test/results/clientpositive/spark/union_remove_10.q.out index 98cad44..1a7ce07 100644 --- ql/src/test/results/clientpositive/spark/union_remove_10.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_10.q.out @@ -89,8 +89,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) - Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0), Reducer 4 (NONE, 0) + Reducer 3 <- Map 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -107,7 +106,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: inputtbl1 @@ -128,7 +127,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 5 + Map 4 Map Operator Tree: TableScan alias: inputtbl1 @@ -142,7 +141,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Reducer 4 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -156,8 +155,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 2 - Vertex: Union 2 Stage: Stage-6 Conditional Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_11.q.out ql/src/test/results/clientpositive/spark/union_remove_11.q.out index 4accb54..257d2ff 100644 --- ql/src/test/results/clientpositive/spark/union_remove_11.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_11.q.out @@ -88,8 +88,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0), Map 4 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -109,7 +107,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: inputtbl1 @@ -126,7 +124,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -143,8 +141,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 2 - Vertex: Union 2 Stage: Stage-6 Conditional Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_15.q.out ql/src/test/results/clientpositive/spark/union_remove_15.q.out index e7b2cd9..55a5ab0 100644 --- ql/src/test/results/clientpositive/spark/union_remove_15.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_15.q.out @@ -75,8 +75,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -100,7 +99,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -138,7 +137,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -155,8 +154,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_16.q.out ql/src/test/results/clientpositive/spark/union_remove_16.q.out index 8838728..d1663b1 100644 --- ql/src/test/results/clientpositive/spark/union_remove_16.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_16.q.out @@ -78,8 +78,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -103,7 +102,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -141,7 +140,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -158,8 +157,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-6 Conditional Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_17.q.out ql/src/test/results/clientpositive/spark/union_remove_17.q.out index 823cbaf..8f317de 100644 --- ql/src/test/results/clientpositive/spark/union_remove_17.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_17.q.out @@ -67,8 +67,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -88,7 +86,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: inputtbl1 @@ -105,8 +103,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_18.q.out ql/src/test/results/clientpositive/spark/union_remove_18.q.out index 28f1b00..91b66ad 100644 --- ql/src/test/results/clientpositive/spark/union_remove_18.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_18.q.out @@ -73,8 +73,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -98,7 +97,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col2 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -136,7 +135,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -153,8 +152,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_19.q.out ql/src/test/results/clientpositive/spark/union_remove_19.q.out index 301aad4..9debf33 100644 --- ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -73,8 +73,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -98,7 +97,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -133,7 +132,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -147,8 +146,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator @@ -265,8 +262,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 3) - Reducer 5 <- Map 4 (GROUP, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -293,7 +289,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -337,7 +333,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -357,8 +353,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator @@ -437,8 +431,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 3) - Reducer 5 <- Map 4 (GROUP, 3) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -462,7 +455,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -502,7 +495,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -521,8 +514,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_2.q.out ql/src/test/results/clientpositive/spark/union_remove_2.q.out index 2b05b7b..1b4fce9 100644 --- ql/src/test/results/clientpositive/spark/union_remove_2.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_2.q.out @@ -75,7 +75,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Map 5 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -99,7 +98,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -113,7 +112,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Map 5 + Map 4 Map Operator Tree: TableScan alias: inputtbl1 @@ -141,8 +140,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_20.q.out ql/src/test/results/clientpositive/spark/union_remove_20.q.out index c67f47b..369a377 100644 --- ql/src/test/results/clientpositive/spark/union_remove_20.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_20.q.out @@ -71,8 +71,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -96,7 +95,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -134,7 +133,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -151,8 +150,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_21.q.out ql/src/test/results/clientpositive/spark/union_remove_21.q.out index 6b119ba..aea9677 100644 --- ql/src/test/results/clientpositive/spark/union_remove_21.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_21.q.out @@ -71,8 +71,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -96,7 +95,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -137,7 +136,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -157,8 +156,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_24.q.out ql/src/test/results/clientpositive/spark/union_remove_24.q.out index 5ed88b4..fb53943 100644 --- ql/src/test/results/clientpositive/spark/union_remove_24.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_24.q.out @@ -67,8 +67,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -92,7 +91,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -130,7 +129,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -147,8 +146,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_25.q.out ql/src/test/results/clientpositive/spark/union_remove_25.q.out index 9445080..d36a246 100644 --- ql/src/test/results/clientpositive/spark/union_remove_25.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_25.q.out @@ -85,8 +85,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -110,7 +109,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -145,7 +144,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -159,8 +158,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator @@ -284,8 +281,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -304,7 +300,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: srcpart @@ -337,7 +333,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Reducer 5 + Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) @@ -354,8 +350,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator @@ -468,8 +462,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -488,7 +481,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Map 4 + Map 3 Map Operator Tree: TableScan alias: srcpart @@ -521,7 +514,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl3 - Reducer 5 + Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col3 (type: string) @@ -538,8 +531,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl3 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_3.q.out ql/src/test/results/clientpositive/spark/union_remove_3.q.out index 09b8636..bac5441 100644 --- ql/src/test/results/clientpositive/spark/union_remove_3.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_3.q.out @@ -73,8 +73,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0), Map 4 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -94,7 +92,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: inputtbl1 @@ -111,7 +109,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -128,8 +126,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 2 - Vertex: Union 2 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_4.q.out ql/src/test/results/clientpositive/spark/union_remove_4.q.out index 65d2aa1..608b111 100644 --- ql/src/test/results/clientpositive/spark/union_remove_4.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_4.q.out @@ -74,8 +74,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -99,7 +98,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -134,7 +133,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -148,8 +147,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-6 Conditional Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_5.q.out ql/src/test/results/clientpositive/spark/union_remove_5.q.out index 41271b2..6f1a49a 100644 --- ql/src/test/results/clientpositive/spark/union_remove_5.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_5.q.out @@ -82,7 +82,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Map 5 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -106,7 +105,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -120,7 +119,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Map 5 + Map 4 Map Operator Tree: TableScan alias: inputtbl1 @@ -148,8 +147,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-6 Conditional Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_6.q.out ql/src/test/results/clientpositive/spark/union_remove_6.q.out index d7cd40b..5c45861 100644 --- ql/src/test/results/clientpositive/spark/union_remove_6.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_6.q.out @@ -72,8 +72,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -97,7 +96,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -139,7 +138,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -160,8 +159,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_7.q.out ql/src/test/results/clientpositive/spark/union_remove_7.q.out index b3be932..d5be4b6 100644 --- ql/src/test/results/clientpositive/spark/union_remove_7.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_7.q.out @@ -73,8 +73,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) + Reducer 4 <- Map 3 (GROUP, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -98,7 +97,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -133,7 +132,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -147,8 +146,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_8.q.out ql/src/test/results/clientpositive/spark/union_remove_8.q.out index 8773535..d7e9aa1 100644 --- ql/src/test/results/clientpositive/spark/union_remove_8.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_8.q.out @@ -79,7 +79,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Map 5 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -103,7 +102,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -117,7 +116,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 5 + Map 4 Map Operator Tree: TableScan alias: inputtbl1 @@ -145,8 +144,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/spark/union_remove_9.q.out ql/src/test/results/clientpositive/spark/union_remove_9.q.out index 8dc6dd8..9157f55 100644 --- ql/src/test/results/clientpositive/spark/union_remove_9.q.out +++ ql/src/test/results/clientpositive/spark/union_remove_9.q.out @@ -86,7 +86,6 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Union 3 <- Map 4 (NONE, 0), Map 5 (NONE, 0), Reducer 2 (NONE, 0) #### A masked pattern was here #### Vertices: Map 1 @@ -110,7 +109,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE value expressions: _col1 (type: bigint) - Map 4 + Map 3 Map Operator Tree: TableScan alias: inputtbl1 @@ -127,7 +126,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Map 5 + Map 4 Map Operator Tree: TableScan alias: inputtbl1 @@ -158,8 +157,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Union 3 - Vertex: Union 3 Stage: Stage-6 Conditional Operator