Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 718751) +++ conf/hive-default.xml (working copy) @@ -102,4 +102,10 @@ How many rows in the right-most join operand Hive should buffer before emitting the join result. + + hive.exec.script.maxerrsize + 100000 + Maximum number of bytes a script is allowed to emit to standard error (per map-reduce task). This prevents runaway scripts from filling logs partitions to capacity + + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 718751) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -47,6 +47,7 @@ PLAN("hive.exec.plan", null), SCRATCHDIR("hive.exec.scratchdir", "/tmp/"+System.getProperty("user.name")+"/hive"), SUBMITVIACHILD("hive.exec.submitviachild", "false"), + SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000), // hadoop stuff HADOOPBIN("hadoop.bin.path", System.getProperty("user.dir") + "/../../../bin/hadoop"), Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (revision 718751) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (working copy) @@ -208,7 +208,10 @@ outThread = new StreamThread(scriptIn, new OutputStreamProcessor( scriptOutputDeserializer.getObjectInspector()), "OutputProcessor"); outThread.start(); - errThread = new StreamThread(scriptErr, new ErrorStreamProcessor (), "ErrorProcessor"); + errThread = new StreamThread(scriptErr, + new ErrorStreamProcessor + (HiveConf.getIntVar(hconf, HiveConf.ConfVars.SCRIPTERRORLIMIT)), + "ErrorProcessor"); errThread.start(); } catch (Exception e) { @@ -297,9 +300,17 @@ } class ErrorStreamProcessor implements StreamProcessor { - public ErrorStreamProcessor () {} + private long bytesCopied = 0; + private long maxBytes; + + public ErrorStreamProcessor (int maxBytes) { + this.maxBytes = (long)maxBytes; + } public void processLine(Text line) throws HiveException { - System.err.println(line.toString()); + if((maxBytes < 0) || (bytesCopied < maxBytes)) { + System.err.println(line.toString()); + } + bytesCopied += line.getLength(); } public void close() { }