Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (revision 956331) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (working copy) @@ -55,6 +55,8 @@ private static final long serialVersionUID = 1L; public static enum Counter {DESERIALIZE_ERRORS} + private transient long maxErrors; + private transient long errors = 0; transient private LongWritable deserialize_error_count = new LongWritable (); transient private Deserializer deserializer; @@ -163,6 +165,7 @@ setConf(mrwork); setChildren(hconf); initialize(hconf, null); + maxErrors = hconf.getLong("hive.max.deserializer.errors", 0L); } private static MapOpCtx initObjectInspector(mapredWork conf, Configuration hconf, String onefile) @@ -350,9 +353,13 @@ forward(rowWithPart, rowObjectInspector); } } catch (SerDeException e) { - // TODO: policy on deserialization errors - deserialize_error_count.set(deserialize_error_count.get()+1); - throw new HiveException (e); + errors++; + deserialize_error_count.set(deserialize_error_count.get()+1); + if (errors > maxErrors) { + throw new HiveException (e); + } else { + LOG.warn("Deserializer error", e); + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (revision 956331) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (working copy) @@ -182,7 +182,7 @@ inputFormats.put(inputFormatClass, newInstance); } catch (Exception e) { throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName() - + " as specified in mapredWork!"); + + " as specified in mapredWork!", e); } } return inputFormats.get(inputFormatClass);