Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (revision 956608) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (working copy) @@ -54,6 +54,8 @@ public class MapOperator extends Operator implements Serializable { private static final long serialVersionUID = 1L; + private transient long maxErrors; + private transient long errors = 0; /** * Counter. @@ -179,6 +181,7 @@ setConf(mrwork); setChildren(hconf); initialize(hconf, null); + maxErrors = hconf.getLong("hive.max.deserializer.errors", 0L); } private static MapOpCtx initObjectInspector(MapredWork conf, @@ -389,17 +392,26 @@ rawRowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2) + " ]"; } - - // TODO: policy on deserialization errors - deserialize_error_count.set(deserialize_error_count.get() + 1); - throw new HiveException("Hive Runtime Error while processing writable " + rawRowString, e); + errors++; + final HiveException exception = new HiveException("Hive Runtime Error while processing writable " + rawRowString, e); + deserialize_error_count.set(deserialize_error_count.get() + 1); + if (errors > maxErrors) { + throw exception; + } else { + LOG.warn(exception.getMessage(), e); + } + throw exception; } try { if (!isPartitioned) { - forward(row, rowObjectInspector); + if (row != null) { + forward(row, rowObjectInspector); + } } else { - forward(rowWithPart, rowObjectInspector); + if (rowWithPart[0] != null) { + forward(rowWithPart, rowObjectInspector); + } } } catch (Exception e) { // Serialize the row and output the error message. Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (revision 956608) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (working copy) @@ -191,7 +191,7 @@ inputFormats.put(inputFormatClass, newInstance); } catch (Exception e) { throw new IOException("Cannot create an instance of InputFormat class " - + inputFormatClass.getName() + " as specified in mapredWork!"); + + inputFormatClass.getName() + " as specified in mapredWork!", e); } } return inputFormats.get(inputFormatClass);