Index: shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java =================================================================== --- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (revision 1979) +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (working copy) @@ -27,6 +27,9 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapred.TaskCompletionEvent; + import java.io.IOException; import java.lang.reflect.Constructor; @@ -306,4 +309,9 @@ return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat"; } + @Override + public Object getTaskAttemptId(TaskCompletionEvent t) { + return t.getTaskAttemptId(); + } + } Index: shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java =================================================================== --- shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java (revision 1979) +++ shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java (working copy) @@ -26,6 +26,8 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskCompletionEvent; + import java.io.IOException; /** @@ -103,4 +105,9 @@ public String getInputFormatClassName() { return "org.apache.hadoop.hive.ql.io.HiveInputFormat"; } + + @Override + public Object getTaskAttemptId(TaskCompletionEvent t) { + return null; + } } Index: shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java =================================================================== --- shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java (revision 1979) +++ shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java (working copy) @@ -26,6 +26,9 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapred.TaskCompletionEvent; + import java.io.IOException; /** @@ -99,8 +102,13 @@ public String getInputFormatClassName() { return "org.apache.hadoop.hive.ql.io.HiveInputFormat"; } - + @Override + public Object getTaskAttemptId(TaskCompletionEvent t) { + return t.getTaskAttemptId(); + } + + @Override public long getAccessTime(FileStatus file) { return -1; } Index: shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java =================================================================== --- shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java (revision 1979) +++ shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java (working copy) @@ -26,6 +26,8 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskCompletionEvent; + import java.io.IOException; /** @@ -104,4 +106,9 @@ public long getAccessTime(FileStatus file) { return file.getAccessTime(); } + + @Override + public Object getTaskAttemptId(TaskCompletionEvent t) { + return t.getTaskAttemptId(); + } } Index: shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java =================================================================== --- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (revision 1979) +++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TaskCompletionEvent; import java.io.IOException; import java.io.DataInput; @@ -108,6 +109,11 @@ public CombineFileInputFormatShim getCombineFileInputFormat(); public String getInputFormatClassName(); + + /** + * getTaskAtemptId is supported starting hadoop 0.18 + */ + public Object getTaskAttemptId(TaskCompletionEvent t); public interface InputSplitShim extends InputSplit { public JobConf getJob(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/JobTrackerURLResolver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/JobTrackerURLResolver.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JobTrackerURLResolver.java (revision 0) @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.net.NetUtils; + +public class JobTrackerURLResolver { + public static String getURL(JobConf conf) throws IOException { + String infoAddr = conf.get("mapred.job.tracker.http.address"); + if (infoAddr == null) { + throw new IOException("Unable to find job tracker info port."); + } + InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr); + int infoPort = infoSocAddr.getPort(); + + String tracker = "http://" + + JobTracker.getAddress(conf).getHostName() + ":" + + infoPort; + + return tracker; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 1979) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -524,6 +525,7 @@ statusMesg += " with errors"; returnVal = 2; console.printError(statusMesg); + showJobFailDebugInfo(job, rj); } else { console.printInfo(statusMesg); } @@ -581,6 +583,75 @@ return (returnVal); } + private void showJobFailDebugInfo(JobConf conf, RunningJob rj) throws IOException { + + Map failures = new HashMap(); + Set successes = new HashSet (); + + int startIndex = 0; + boolean more = true; + + while(more) { + TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex); + + if(taskCompletions == null || taskCompletions.length == 0) { + more = false; + break; + } + + for(TaskCompletionEvent t : taskCompletions) { + TaskAttemptID attemptId = (TaskAttemptID)ShimLoader.getHadoopShims().getTaskAttemptId(t); + if(attemptId == null) { + console.printError("Task attempt info is unavailable in this Hadoop version"); + more = false; + break; + } + + if(t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) { + Integer failAttempts = failures.get(attemptId.getTaskID()); + if(failAttempts == null) { + failAttempts = new Integer(0); + } + failAttempts = failAttempts.intValue() + 1; + failures.put(attemptId.getTaskID(), failAttempts); + } else { + successes.add(attemptId.getTaskID()); + } + } + + startIndex += taskCompletions.length; + } + // Remove failures for tasks that succeeded + for(TaskID task : successes) { + failures.remove(task); + } + + if(failures.keySet().size() == 0) { + return; + } + + // Find the highest failure count + int maxFailures = 0; + for(Integer failCount : failures.values()) { + if(maxFailures < failCount.intValue()) + maxFailures = failCount.intValue(); + } + + // Display Error Message for tasks with the highest failure count + console.printError("\nFailed tasks with most" + "(" + maxFailures + ")" + " failures " + ": "); + String jtUrl = JobTrackerURLResolver.getURL(conf); + + for(TaskID task : failures.keySet()) { + if(failures.get(task).intValue() == maxFailures) { + String jobId = task.getJobID().toString(); + String taskUrl = jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString(); + console.printError("Task URL: " + taskUrl +"\n"); + } + } + return; + + } + private static void printUsage() { System.out .println("ExecDriver -plan [-jobconf k1=v1 [-jobconf k2=v2] ...] " Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Throttle.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Throttle.java (revision 1979) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Throttle.java (working copy) @@ -65,18 +65,8 @@ return; } - // find the http port for the jobtracker - String infoAddr = conf.get("mapred.job.tracker.http.address"); - if (infoAddr == null) { - throw new IOException("Throttle: Unable to find job tracker info port."); - } - InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr); - int infoPort = infoSocAddr.getPort(); - // This is the Job Tracker URL - String tracker = "http://" + - JobTracker.getAddress(conf).getHostName() + ":" + - infoPort + + String tracker = JobTrackerURLResolver.getURL(conf) + "/gc.jsp?threshold=" + threshold; while (true) {