Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 718751)
+++ conf/hive-default.xml (working copy)
@@ -102,4 +102,10 @@
How many rows in the right-most join operand Hive should buffer before emitting the join result.
+
+ hive.exec.script.maxerrsize
+ 100000
+ Maximum number of bytes a script is allowed to emit to standard error (per map-reduce task). This prevents runaway scripts from filling logs partitions to capacity
+
+
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 718751)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -47,6 +47,7 @@
PLAN("hive.exec.plan", null),
SCRATCHDIR("hive.exec.scratchdir", "/tmp/"+System.getProperty("user.name")+"/hive"),
SUBMITVIACHILD("hive.exec.submitviachild", "false"),
+ SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000),
// hadoop stuff
HADOOPBIN("hadoop.bin.path", System.getProperty("user.dir") + "/../../../bin/hadoop"),
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (revision 718751)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (working copy)
@@ -208,7 +208,10 @@
outThread = new StreamThread(scriptIn, new OutputStreamProcessor(
scriptOutputDeserializer.getObjectInspector()), "OutputProcessor");
outThread.start();
- errThread = new StreamThread(scriptErr, new ErrorStreamProcessor (), "ErrorProcessor");
+ errThread = new StreamThread(scriptErr,
+ new ErrorStreamProcessor
+ (HiveConf.getIntVar(hconf, HiveConf.ConfVars.SCRIPTERRORLIMIT)),
+ "ErrorProcessor");
errThread.start();
} catch (Exception e) {
@@ -297,9 +300,17 @@
}
class ErrorStreamProcessor implements StreamProcessor {
- public ErrorStreamProcessor () {}
+ private long bytesCopied = 0;
+ private long maxBytes;
+
+ public ErrorStreamProcessor (int maxBytes) {
+ this.maxBytes = (long)maxBytes;
+ }
public void processLine(Text line) throws HiveException {
- System.err.println(line.toString());
+ if((maxBytes < 0) || (bytesCopied < maxBytes)) {
+ System.err.println(line.toString());
+ }
+ bytesCopied += line.getLength();
}
public void close() {
}