Index: build-common.xml =================================================================== --- build-common.xml +++ build-common.xml @@ -59,7 +59,7 @@ - + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.Serializable; +import java.lang.Exception; +import java.net.MalformedURLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -106,10 +108,6 @@ return "Ended Job = " + jobId; } - private String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) { - return taskTrackerHttpAddress + "/tasklog?taskid=" + taskAttemptId + "&all=true"; - } - public boolean mapStarted() { return mapProgress > 0; } @@ -492,7 +490,8 @@ } @SuppressWarnings("deprecation") - private void showJobFailDebugInfo(JobConf conf, RunningJob rj) throws IOException { + private void showJobFailDebugInfo(JobConf conf, RunningJob rj) + throws IOException, MalformedURLException { // Mapping from task ID to the number of failures Map failures = new HashMap(); // Successful task ID's @@ -541,7 +540,8 @@ } // These tasks should have come from the same job. assert (ti.getJobId() != null && ti.getJobId().equals(jobId)); - ti.getLogUrls().add(getTaskAttemptLogUrl(t.getTaskTrackerHttp(), t.getTaskId())); + ti.getLogUrls().add(ShimLoader.getHadoopShims().getTaskAttemptLogUrl( + t.getTaskTrackerHttp(), t.getTaskId())); // If a task failed, then keep track of the total number of failures // for that task (typically, a task gets re-run up to 4 times if it Index: ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.exec; import java.io.IOException; +import java.lang.Exception; +import java.net.MalformedURLException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -94,21 +96,18 @@ console.printError(e.getMessage()); } } - private String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) { - return taskTrackerHttpAddress + "/tasklog?taskid=" + taskAttemptId + "&start=-8193"; - } class TaskLogGrabber implements Runnable { public void run() { try { getTaskLogs(); - } catch (IOException e) { + } catch (Exception e) { console.printError(e.getMessage()); } } - private void getTaskLogs() throws IOException { + private void getTaskLogs() throws IOException, MalformedURLException { int startIndex = 0; while (true) { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex); @@ -146,7 +145,8 @@ } // These tasks should have come from the same job. assert (ti.getJobId() != null && ti.getJobId().equals(jobId)); - ti.getLogUrls().add(getTaskAttemptLogUrl(t.getTaskTrackerHttp(), t.getTaskId())); + ti.getLogUrls().add(ShimLoader.getHadoopShims().getTaskAttemptLogUrl( + t.getTaskTrackerHttp(), t.getTaskId())); // If a task failed, then keep track of the total number of failures // for that task (typically, a task gets re-run up to 4 times if it Index: ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java =================================================================== --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import java.lang.NullPointerException; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; + +/** + * GenericUDFEvaluateNPE + * This UDF is to throw an Null Pointer Exception + * It is used to test hive failure handling + * + */ +@Description(name = "evaluate_npe", value = "_FUNC_(string) - Throws an Null Pointer Exception") +public class GenericUDFEvaluateNPE extends GenericUDF { + private ObjectInspector[] argumentOIs; + private final Text result= new Text(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "The function evaluate_npe(string)" + + "needs only one argument."); + } + + if (!arguments[0].getTypeName().equals(Constants.STRING_TYPE_NAME)) { + throw new UDFArgumentTypeException(0, + "Argument 1 of function evaluate_npe must be \"" + + Constants.STRING_TYPE_NAME + "but \"" + + arguments[0].getTypeName() + "\" was found."); + } + + argumentOIs = arguments; + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (true) { + throw new NullPointerException("evaluate null pointer exception"); + } + return result; + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + return "evaluate_npe(" + children[0] + ")"; + } +} Index: ql/src/test/queries/clientnegative/cluster_tasklog_retrieval.q =================================================================== --- /dev/null +++ ql/src/test/queries/clientnegative/cluster_tasklog_retrieval.q @@ -0,0 +1,6 @@ +-- TaskLog retrieval upon Null Pointer Exception in Cluster + +CREATE TEMPORARY FUNCTION evaluate_npe AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'; + +FROM src +SELECT evaluate_npe(src.key) LIMIT 1; Index: ql/src/test/results/clientnegative/cluster_tasklog_retrieval.q.out =================================================================== --- /dev/null +++ ql/src/test/results/clientnegative/cluster_tasklog_retrieval.q.out @@ -0,0 +1,14 @@ +PREHOOK: query: -- TaskLog retrieval upon Null Pointer Exception in Cluster + +CREATE TEMPORARY FUNCTION evaluate_npe AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +PREHOOK: type: CREATEFUNCTION +POSTHOOK: query: -- TaskLog retrieval upon Null Pointer Exception in Cluster + +CREATE TEMPORARY FUNCTION evaluate_npe AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +POSTHOOK: type: CREATEFUNCTION +PREHOOK: query: FROM src +SELECT evaluate_npe(src.key) LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask Index: shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java =================================================================== --- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -21,6 +21,7 @@ import java.io.DataOutput; import java.io.IOException; import java.lang.reflect.Constructor; +import java.net.MalformedURLException; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -526,6 +527,14 @@ } @Override + public String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) + throws MalformedURLException { + // In hadoop 0.20, no utility function is available to construct task log url + // Manually constructing the URL here + return taskTrackerHttpAddress + "/tasklog?taskid=" + taskAttemptId + "&all=true"; + } + + @Override public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception { JobTrackerState state; switch (clusterStatus.getJobTrackerState()) { Index: shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java =================================================================== --- shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.shims; +import java.net.MalformedURLException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.shims.HadoopShimsSecure; import org.apache.hadoop.mapred.ClusterStatus; @@ -30,6 +32,14 @@ public class Hadoop20SShims extends HadoopShimsSecure { @Override + public String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) + throws MalformedURLException { + // In hadoop 0.20, no utility function is available to construct task log url + // Manually constructing the URL here + return taskTrackerHttpAddress + "/tasklog?taskid=" + taskAttemptId + "&all=true"; + } + + @Override public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception { JobTrackerState state; switch (clusterStatus.getJobTrackerState()) { Index: shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java =================================================================== --- shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -25,14 +25,28 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.util.Progressable; +import java.net.URL; +import java.lang.Integer; +import java.net.MalformedURLException; + /** * Implemention of shims against Hadoop 0.23.0. */ public class Hadoop23Shims extends HadoopShimsSecure { @Override + public String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) + throws MalformedURLException { + URL taskTrackerHttpURL = new URL(taskTrackerHttpAddress); + return HostUtil.getTaskLogUrl(taskTrackerHttpURL.getHost(), + Integer.toString(taskTrackerHttpURL.getPort()), + taskAttemptId); + } + + @Override public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception { JobTrackerState state; switch (clusterStatus.getJobTrackerStatus()) { Index: shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java =================================================================== --- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -20,6 +20,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.net.MalformedURLException; import java.security.PrivilegedExceptionAction; import java.util.List; @@ -66,6 +67,14 @@ boolean usesJobShell(); /** + * Constructs and Returns TaskAttempt Log Url + * + * @return TaskAttempt Log Url + */ + String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) + throws MalformedURLException; + + /** * Return true if the job has not switched to RUNNING state yet * and is still in PREP state */