diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index 0a0c791..62237e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -129,7 +129,6 @@ public final Map, BaseWork> unionWorkMap; public final List currentUnionOperators; public final Set workWithUnionOperators; - public final Set clonedReduceSinks; public final Set fileSinkSet; public final Map> fileSinkMap; @@ -180,7 +179,6 @@ public GenSparkProcContext(HiveConf conf, this.unionWorkMap = new LinkedHashMap, BaseWork>(); this.currentUnionOperators = new LinkedList(); this.workWithUnionOperators = new LinkedHashSet(); - this.clonedReduceSinks = new LinkedHashSet(); this.fileSinkSet = new LinkedHashSet(); this.fileSinkMap = new LinkedHashMap>(); this.pruningSinkSet = new LinkedHashSet>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java index 3dd6d92..2d5bb62 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java @@ -94,12 +94,6 @@ public Object process(Node nd, Stack stack, LOG.debug("Root operator: " + root); LOG.debug("Leaf operator: " + operator); - if (context.clonedReduceSinks.contains(operator)) { - // if we're visiting a terminal we've created ourselves, - // just skip and keep going - return null; - } - SparkWork sparkWork = context.currentTask.getWork(); SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class); @@ -192,7 +186,6 @@ public Object process(Node nd, Stack stack, // we've already set this one up. Need to clone for the next work. r = (ReduceSinkOperator) OperatorFactory.getAndMakeChild( (ReduceSinkDesc)r.getConf().clone(), r.getParentOperators()); - context.clonedReduceSinks.add(r); } r.getConf().setOutputName(work.getName()); }