diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java index 7df626b..0ea96ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java @@ -70,9 +70,9 @@ private void setupMRLegacyConfigs() { taskAttemptIdBuilder.append("r_"); } - // Spark task attempt id is increased by Spark context instead of task, which may introduce - // unstable qtest output, since non Hive features depends on this, we always set it to 0 here. - taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().partitionId())) + // Hive relies on this unique id as the FileSinkOp's output filename. Retries of the same + // task will have the same parititionId while with different taskAttemptId. + taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().taskAttemptId())) .append("_0"); String taskAttemptIdStr = taskAttemptIdBuilder.toString();