Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 4486) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -120,6 +120,9 @@ HIVEPARTITIONNAME("hive.partition.name", ""), HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", false), HIVESCRIPTIDENVVAR("hive.script.operator.id.env.var", "HIVE_SCRIPT_OPERATOR_ID"), + // The maximum size in bytes that TextRecordReader will read in per call + // to next(). Set to 10MB so that the limit is generally not encountered. + HIVETEXTRECORDREADERMAXSIZE("hive.text.record.reader.max.length", 10*1024*1024), HIVEMAPREDMODE("hive.mapred.mode", "nonstrict"), HIVEALIAS("hive.alias", ""), HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"), Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java (revision 4486) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java (working copy) @@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.LineRecordReader.LineReader; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -33,10 +34,13 @@ private LineReader lineReader; private InputStream in; private Text row; - + private int maxConsumeSize; + public void initialize(InputStream in, Configuration conf) throws IOException { lineReader = new LineReader(in, conf); this.in = in; + this.maxConsumeSize = HiveConf.getIntVar(conf, + HiveConf.ConfVars.HIVETEXTRECORDREADERMAXSIZE); } public Writable createRow() throws IOException { @@ -47,8 +51,7 @@ public int next(Writable row) throws IOException { if (lineReader == null) return -1; - - return lineReader.readLine((Text)row); + return lineReader.readLine((Text)row, Integer.MAX_VALUE, maxConsumeSize); } public void close() throws IOException {