diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index ab539c4..80cfc98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -390,7 +390,36 @@ public static FooterInfo extractMetaInfoFromFooter( bb.limit(footerAbsPos + footerSize); InputStream instream = InStream.create("footer", Lists.newArrayList( new BufferChunk(bb, 0)), footerSize, codec, bufferSize); - return OrcProto.Footer.parseFrom(instream); + CodedInputStream in = CodedInputStream.newInstance(instream); + int msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT; + OrcProto.Footer footer = null; + do { + try { + in.setSizeLimit(msgLimit); + footer = OrcProto.Footer.parseFrom(in); + } catch (InvalidProtocolBufferException e) { + if (e.getMessage().contains("Protocol message was too large")) { + LOG.warn("Footer section is larger than " + msgLimit + " bytes. Increasing the max" + + " size of the coded input stream." ); + + msgLimit = msgLimit << 1; + if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) { + LOG.error("Footer section exceeds max protobuf message size of " + + PROTOBUF_MESSAGE_MAX_LIMIT + " bytes."); + throw e; + } + + // we must have failed in the middle of reading instream and instream doesn't support + // resetting the stream + instream = InStream.create("footer", Lists.newArrayList( + new BufferChunk(bb, 0)), footerSize, codec, bufferSize); + in = CodedInputStream.newInstance(instream); + } else { + throw e; + } + } + } while (footer == null); + return footer; } private static OrcProto.Metadata extractMetadata(ByteBuffer bb, int metadataAbsPos,