diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java index 7cfd43d..69c8725 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java @@ -42,7 +42,7 @@ private void setupMRLegacyConfigs() { StringBuilder taskAttemptIdBuilder = new StringBuilder("attempt_"); taskAttemptIdBuilder.append(System.currentTimeMillis()) .append("_") - .append(stageIdFormat.format(TaskContext.get().getStageId())) + .append(stageIdFormat.format(TaskContext.get().stageId())) .append("_"); if (isMap()) { @@ -53,12 +53,12 @@ private void setupMRLegacyConfigs() { // Spark task attempt id is increased by Spark context instead of task, which may introduce // unstable qtest output, since non Hive features depends on this, we always set it to 0 here. - taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().getPartitionId())) + taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().partitionId())) .append("_0"); String taskAttemptIdStr = taskAttemptIdBuilder.toString(); jobConf.set("mapred.task.id", taskAttemptIdStr); jobConf.set("mapreduce.task.attempt.id", taskAttemptIdStr); - jobConf.setInt("mapred.task.partition", TaskContext.get().getPartitionId()); + jobConf.setInt("mapred.task.partition", TaskContext.get().partitionId()); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java index 3773dcb..5bede0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java @@ -26,11 +26,11 @@ public class SparkUtilities { public static void setTaskInfoInJobConf(JobConf jobConf, TaskContext taskContext) { // Set mapred.task.partition in executor side. - jobConf.setInt("mapred.task.partition", taskContext.getPartitionId()); + jobConf.setInt("mapred.task.partition", taskContext.partitionId()); // Set mapred.task.id as taskId_attemptId. The taskId is 6 digits in length (prefixed with 0 if // necessary). Similarly attemptId is two digits in length. jobConf.set("mapred.task.id", - String.format("%06d_%02d", taskContext.getPartitionId(), taskContext.getAttemptId())); + String.format("%06d_%02d", taskContext.partitionId(), taskContext.attemptId())); } }