From b1d49561954199f8075e16b94fc1627b74e4724a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 24 Jun 2019 20:16:28 +0800 Subject: [PATCH] Fix tez + union all + udtf data loss issue --- .../java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 11c1df6806..be29886a26 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -293,7 +293,11 @@ public void removeUnionOperators(Configuration conf, GenTezProcContext context, FileSinkOperator fileSink = (FileSinkOperator)current; // remember it for additional processing later - context.fileSinkSet.add(fileSink); + if (context.fileSinkSet.contains(fileSink)) { + continue; + } else { + context.fileSinkSet.add(fileSink); + } FileSinkDesc desc = fileSink.getConf(); Path path = desc.getDirName(); -- 2.20.1.windows.1