diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java index 5199071..6935738 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java @@ -390,17 +390,4 @@ public StageType getType() { public String getName() { return "COLUMNSTATS TASK"; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - FetchWork fWork = work.getfWork(); - String s = fWork.getTblDir(); - if ((s != null) && ctx.isMRTmpFileURI(s)) { - fWork.setTblDir(ctx.localizeMRTmpFileURI(s)); - } - ArrayList ls = fWork.getPartDir(); - if (ls != null) { - ctx.localizePaths(ls); - } - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java index 60d648b..854cd52 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java @@ -205,15 +205,6 @@ public boolean addDependentTask(Task dependent) { } @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - if (getListTasks() != null) { - for (Task t : getListTasks()) { - t.localizeMRTmpFiles(ctx); - } - } - } - - @Override public List> getDependentTasks() { return listTasks; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java index 23bebc5..38d97e3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java @@ -106,12 +106,4 @@ public StageType getType() { public String getName() { return "COPY"; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // copy task is only used by the load command and - // does not use any map-reduce tmp files - // we don't expect to enter this code path at all - throw new RuntimeException ("Unexpected call"); - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 939defc..295daab 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4059,9 +4059,4 @@ public StageType getType() { public String getName() { return "DDL"; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // no-op - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java index 7c38c34..9189cfc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java @@ -53,12 +53,4 @@ public StageType getType() { public String getName() { return "DEPENDENCY_COLLECTION"; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // copy task doesn't have any execution and so - // does not use any map-reduce tmp files - // we don't expect to enter this code path at all - throw new RuntimeException ("Unexpected call"); - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java index 4da68a0..11772e6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java @@ -915,57 +915,6 @@ public String getName() { } @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - - // localize any map-reduce input paths - ctx.localizeKeys((Map) ((Object) work.getPathToAliases())); - ctx.localizeKeys((Map) ((Object) work.getPathToPartitionInfo())); - - // localize any input paths for maplocal work - MapredLocalWork l = work.getMapLocalWork(); - if (l != null) { - Map m = l.getAliasToFetchWork(); - if (m != null) { - for (FetchWork fw : m.values()) { - String s = fw.getTblDir(); - if ((s != null) && ctx.isMRTmpFileURI(s)) { - fw.setTblDir(ctx.localizeMRTmpFileURI(s)); - } - } - } - } - - // fix up outputs - Map> pa = work.getPathToAliases(); - if (pa != null) { - for (List ls : pa.values()) { - for (String a : ls) { - ArrayList> opList = - new ArrayList>(); - opList.add(work.getAliasToWork().get(a)); - - while (!opList.isEmpty()) { - Operator op = opList.remove(0); - - if (op instanceof FileSinkOperator) { - FileSinkDesc fdesc = ((FileSinkOperator) op).getConf(); - String s = fdesc.getDirName(); - if ((s != null) && ctx.isMRTmpFileURI(s)) { - fdesc.setDirName(ctx.localizeMRTmpFileURI(s)); - } - ((FileSinkOperator) op).setConf(fdesc); - } - - if (op.getChildOperators() != null) { - opList.addAll(op.getChildOperators()); - } - } - } - } - } - } - - @Override public void updateCounters(Counters ctrs, RunningJob rj) throws IOException { for (Operator op : work.getAliasToWork().values()) { op.updateCounters(ctrs); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 5981f84..5a00c2d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -717,13 +717,6 @@ public String getName() { } @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // explain task has nothing to localize - // we don't expect to enter this code path at all - throw new RuntimeException("Unexpected call"); - } - - @Override public List getResultSchema() { FieldSchema tmpFieldSchema = new FieldSchema(); List colList = new ArrayList(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java index c76c37d..6e9e0a8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java @@ -163,19 +163,6 @@ public String getName() { return "FETCH"; } - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - String s = work.getTblDir(); - if ((s != null) && ctx.isMRTmpFileURI(s)) { - work.setTblDir(ctx.localizeMRTmpFileURI(s)); - } - - ArrayList ls = work.getPartDir(); - if (ls != null) { - ctx.localizePaths(ls); - } - } - /** * Clear the Fetch Operator. * diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java index 2905ff5..1d38f07 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java @@ -114,9 +114,4 @@ public StageType getType() { public String getName() { return "FUNCTION"; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - throw new RuntimeException ("Unexpected call"); - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java index dd0efc7..6bbcb26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java @@ -448,11 +448,6 @@ private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String cur } @Override - public void localizeMRTmpFilesImpl(Context ctx) { - - } - - @Override public boolean isMapRedLocalTask() { return true; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 4f4d1fb..ac8e167 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -537,10 +537,4 @@ public StageType getType() { public String getName() { return "MOVE"; } - - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // no-op - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index 69e2d3d..599f63c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -263,11 +263,6 @@ public String getName() { return "STATS"; } - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // Nothing to do for StatsTask here. - } - private int aggregateStats() { StatsAggregator statsAggregator = null; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java index 99fab34..17387a9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java @@ -384,37 +384,6 @@ public boolean hasReduce() { public abstract StageType getType(); /** - * If this task uses any map-reduce intermediate data (either for reading or for writing), - * localize them (using the supplied Context). Map-Reduce intermediate directories are allocated - * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI(). - * - * This method is declared abstract to force any task code to explicitly deal with this aspect of - * execution. - * - * @param ctx - * context object with which to localize - */ - abstract protected void localizeMRTmpFilesImpl(Context ctx); - - /** - * Localize a task tree - * - * @param ctx - * context object with which to localize - */ - public final void localizeMRTmpFiles(Context ctx) { - localizeMRTmpFilesImpl(ctx); - - if (childTasks == null) { - return; - } - - for (Task t : childTasks) { - t.localizeMRTmpFiles(ctx); - } - } - - /** * Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its ancestor. * Feed is a generic form of execution-time feedback (type, value) pair from one task to another * task. Examples include dynamic partitions (which are only available at execution time). The diff --git ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java index 45f3b7e..364fc19 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java @@ -98,9 +98,4 @@ public String getName() { public StageType getType() { return StageType.DDL; } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java index ee667ce..cbee423 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java @@ -378,10 +378,4 @@ public void logPlanProgress(SessionState ss) throws IOException { public void updateCounters(Counters ctrs, RunningJob rj) throws IOException { // no op } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // no op - } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java index 767cb02..a1abf90 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java @@ -379,11 +379,4 @@ public void logPlanProgress(SessionState ss) throws IOException { public void updateCounters(Counters ctrs, RunningJob rj) throws IOException { // no op } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // no op - } - - } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java index e9805bb..a9cd8ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java @@ -254,10 +254,4 @@ public void logPlanProgress(SessionState ss) throws IOException { public void updateCounters(Counters ctrs, RunningJob rj) throws IOException { // no op } - - @Override - protected void localizeMRTmpFilesImpl(Context ctx) { - // no op - } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d115f24..6f74763 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -9637,15 +9637,6 @@ public boolean accept(Path file) { ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(conf)); ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local"); console.printInfo("Automatically selecting local only mode for query"); - - // If all the tasks can be run locally, we can use local disk for - // storing intermediate data. - - /** - * This code is commented out pending further testing/development - * for (Task t: rootTasks) - * t.localizeMRTmpFiles(ctx); - */ } }