diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java index 0d0770f..f2acd75 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java @@ -272,8 +272,8 @@ private void processAllEvents(String inputName, int count = 0; for (InputSplit inputSplit : finalSplits) { MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit); - InputDataInformationEvent diEvent = - InputDataInformationEvent.create(count, serializedSplit.toByteString().asReadOnlyByteBuffer()); + InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload( + count, serializedSplit.toByteString().asReadOnlyByteBuffer()); diEvent.setTargetIndex(count); count++; taskEvents.add(diEvent); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 90ce01e..609adea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -505,7 +505,7 @@ private Vertex createVertex(JobConf conf, MapWork mapWork, localResources.put(getBaseName(lr), lr); } - map.setTaskLocalFiles(localResources); + map.addTaskLocalFiles(localResources); return map; } @@ -554,7 +554,7 @@ private Vertex createVertex(JobConf conf, ReduceWork reduceWork, for (LocalResource lr: additionalLr) { localResources.put(getBaseName(lr), lr); } - reducer.setTaskLocalFiles(localResources); + reducer.addTaskLocalFiles(localResources); return reducer; } @@ -607,7 +607,7 @@ public PreWarmVertex createPreWarmVertex(TezConfiguration conf, combinedResources.putAll(localResources); } - prewarmVertex.setTaskLocalFiles(localResources); + prewarmVertex.addTaskLocalFiles(localResources); prewarmVertex.setTaskLaunchCmdOpts(getContainerJavaOpts(conf)); prewarmVertex.setTaskEnvironment(getContainerEnvironment(conf, false)); return prewarmVertex; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index f9fbf22..d540e27 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -207,14 +207,15 @@ public void handleInputInitializerEvent(List events) thro MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto(); int count = 0; for (MRSplitProto mrSplit : splitsProto.getSplitsList()) { - InputDataInformationEvent diEvent = - InputDataInformationEvent.create(count++, mrSplit.toByteString().asReadOnlyByteBuffer()); + InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload( + count++, mrSplit.toByteString().asReadOnlyByteBuffer()); events.add(diEvent); } } else { int count = 0; for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) { - InputDataInformationEvent diEvent = InputDataInformationEvent.create(count++, split); + InputDataInformationEvent diEvent = InputDataInformationEvent.createWithObjectPayload( + count++, split); events.add(diEvent); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index e1f5630..94c0653 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -66,7 +66,7 @@ * using the Tez APIs directly. * */ -@SuppressWarnings({"serial", "deprecation"}) +@SuppressWarnings({"serial"}) public class TezTask extends Task { private static final String CLASS_NAME = TezTask.class.getName(); @@ -302,7 +302,7 @@ DAGClient submit(JobConf conf, DAG dag, Path scratchDir, try { // ready to start execution on the cluster - sessionState.getSession().addAppMasterLocalResources(resourceMap); + sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening...");