diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index 5ddc16d..4211a07 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.DependencyCollectionTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -45,7 +44,6 @@ import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.SparkEdgeProperty; import org.apache.hadoop.hive.ql.plan.SparkWork; @@ -125,8 +123,6 @@ public final Set workWithUnionOperators; public final Set clonedReduceSinks; - // we link filesink that will write to the same final location - public final Map> linkedFileSinks; public final Set fileSinkSet; // remember which reducesinks we've already connected @@ -159,7 +155,6 @@ public GenSparkProcContext(HiveConf conf, ParseContext parseContext, this.currentUnionOperators = new LinkedList(); this.workWithUnionOperators = new LinkedHashSet(); this.clonedReduceSinks = new LinkedHashSet(); - this.linkedFileSinks = new LinkedHashMap>(); this.fileSinkSet = new LinkedHashSet(); this.connectedReduceSinks = new LinkedHashSet(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java index 3cdfc51..695d8b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java @@ -213,22 +213,6 @@ public void removeUnionOperators(Configuration conf, GenSparkProcContext context Operator current = operators.pop(); seen.add(current); - if (current instanceof FileSinkOperator) { - FileSinkOperator fileSink = (FileSinkOperator)current; - - FileSinkDesc desc = fileSink.getConf(); - Path path = desc.getDirName(); - List linked; - - if (!context.linkedFileSinks.containsKey(path)) { - linked = new ArrayList(); - context.linkedFileSinks.put(path, linked); - } - linked = context.linkedFileSinks.get(path); - linked.add(desc); - desc.setLinkedFileSinkDesc(linked); - } - if (current instanceof UnionOperator) { Operator parent = null; int count = 0;