diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index c990d85..232ab65 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -506,8 +506,37 @@ private static FileMetaInfo extractMetaInfoFromFooter(FileSystem fs, footerBuffer.limit(position + metadataSize + footerBufferSize); instream = InStream.create("footer", Lists.newArrayList( new BufferChunk(footerBuffer, 0)), footerBufferSize, codec, bufferSize); - this.footer = OrcProto.Footer.parseFrom(instream); + in = CodedInputStream.newInstance(instream); + msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT; + OrcProto.Footer footerProto = null; + do { + try { + in.setSizeLimit(msgLimit); + footerProto = OrcProto.Footer.parseFrom(in); + } catch (InvalidProtocolBufferException e) { + if (e.getMessage().contains("Protocol message was too large")) { + LOG.warn("Footer section is larger than " + msgLimit + " bytes. Increasing the max" + + " size of the coded input stream."); + + msgLimit = msgLimit << 1; + if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) { + LOG.error("Footer section exceeds max protobuf message size of " + + PROTOBUF_MESSAGE_MAX_LIMIT + " bytes."); + throw e; + } + + // we must have failed in the middle of reading instream and instream doesn't support + // resetting the stream + instream = InStream.create("footer", Lists.newArrayList( + new BufferChunk(footerBuffer, 0)), footerBufferSize, codec, bufferSize); + in = CodedInputStream.newInstance(instream); + } else { + throw e; + } + } + } while (footerProto == null); + this.footer = footerProto; footerBuffer.position(position); this.inspector = OrcStruct.createObjectInspector(0, footer.getTypesList()); }