Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (revision 1517792) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (working copy) @@ -799,7 +799,16 @@ freqReader.reset(docFreqBytes, 0, len); // metadata - len = in.readVInt(); + if (postingsReader.pulsed()) { + int start = (int)in.getFilePointer(); + len = in.readVInt(); + in.seek(in.getFilePointer() + len); + len = in.readVInt(); + len = len - start + (int)in.getFilePointer(); + in.seek(start); + } else { + len = in.readVInt(); + } if (bytes == null) { bytes = new byte[ArrayUtil.oversize(len, 1)]; bytesReader = new ByteArrayDataInput(); Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (revision 1517792) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (working copy) @@ -342,6 +342,7 @@ // 4th pass: write the metadata long[] longs = new long[longsSize]; boolean absolute = true; + pendingTerms[0].state.termBlockOrd = pendingCount; for(int termCount=0;termCount= 0 ? termState.totalTermFreq : termState.docFreq; @@ -167,7 +194,7 @@ // Inlined into terms dict -- just read the byte[] blob in, // but don't decode it now (we only decode when a DocsEnum // or D&PEnum is pulled): - termState.postingsSize = in.readVInt(); + termState.postingsSize = termState.inlinedReader.readVInt(); if (termState.postings == null || termState.postings.length < termState.postingsSize) { termState.postings = new byte[ArrayUtil.oversize(termState.postingsSize, 1)]; } @@ -175,7 +202,7 @@ // (the blob holding all inlined terms' blobs for // current term block) into another byte[] (just the // blob for this term)... - in.readBytes(termState.postings, 0, termState.postingsSize); + termState.inlinedReader.readBytes(termState.postings, 0, termState.postingsSize); //System.out.println(" inlined bytes=" + termState.postingsSize); termState.absolute = termState.absolute || absolute; } else { @@ -185,14 +212,15 @@ termState.longs = new long[longsSize]; } for (int i = 0; i < longsSize; i++) { - termState.longs[i] = in.readVLong(); + termState.longs[i] = termState.wrappedReader.readVLong(); } termState.postingsSize = -1; termState.wrappedTermState.docFreq = termState.docFreq; termState.wrappedTermState.totalTermFreq = termState.totalTermFreq; - wrappedPostingsReader.decodeTerm(termState.longs, in, fieldInfo, termState.wrappedTermState, termState.absolute); + wrappedPostingsReader.decodeTerm(termState.longs, termState.wrappedReader, fieldInfo, termState.wrappedTermState, termState.absolute); termState.absolute = false; } } @Override Index: lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java (revision 1517792) +++ lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java (working copy) @@ -82,6 +82,7 @@ private static class PulsingTermState extends BlockTermState { private byte[] bytes; private BlockTermState wrappedState; + @Override public String toString() { if (bytes != null) { @@ -147,6 +148,11 @@ } @Override + public boolean pulsed() { + return true; + } + + @Override public BlockTermState newTermState() throws IOException { PulsingTermState state = new PulsingTermState(); state.wrappedState = wrappedPostingsWriter.newTermState(); @@ -379,24 +385,46 @@ pendingCount = 0; } + private int termBlockCount = 0; + private int termBlockOrd = 0; + private final RAMOutputStream inlinedBuffer = new RAMOutputStream(); + private final RAMOutputStream wrappedBuffer = new RAMOutputStream(); + @Override public void encodeTerm(long[] empty, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException { PulsingTermState state = (PulsingTermState)_state; + if (absolute) { + termBlockCount = state.termBlockOrd; + termBlockOrd = 0; + } assert empty.length == 0; this.absolute = this.absolute || absolute; if (state.bytes == null) { wrappedPostingsWriter.encodeTerm(longs, buffer, fieldInfo, state.wrappedState, this.absolute); for (int i = 0; i < longsSize; i++) { - out.writeVLong(longs[i]); + wrappedBuffer.writeVLong(longs[i]); } - buffer.writeTo(out); + buffer.writeTo(wrappedBuffer); buffer.reset(); this.absolute = false; } else { - out.writeVInt(state.bytes.length); - out.writeBytes(state.bytes, 0, state.bytes.length); + inlinedBuffer.writeVInt(state.bytes.length); + inlinedBuffer.writeBytes(state.bytes, 0, state.bytes.length); this.absolute = this.absolute || absolute; } + termBlockOrd++; + if (termBlockOrd == termBlockCount) { + out.writeVInt((int)inlinedBuffer.getFilePointer()); + inlinedBuffer.writeTo(out); + out.writeVInt((int)wrappedBuffer.getFilePointer()); + wrappedBuffer.writeTo(out); + inlinedBuffer.reset(); + wrappedBuffer.reset(); + } } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1517792) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy) @@ -732,7 +732,18 @@ nextEnt = 0; // metadata - numBytes = in.readVInt(); + if (postingsReader.pulsed()) { + int start = (int)in.getFilePointer(); + numBytes = in.readVInt(); + in.seek(in.getFilePointer() + numBytes); + numBytes = in.readVInt(); + numBytes = numBytes - start + (int)in.getFilePointer(); + in.seek(start); + } else { + numBytes = in.readVInt(); + } if (bytes == null) { bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; bytesReader = new ByteArrayDataInput(); @@ -741,6 +752,7 @@ } in.readBytes(bytes, 0, numBytes); bytesReader.reset(bytes, 0, numBytes); if (!isLastInFloor) { // Sub-blocks of a single floor block are always @@ -2423,7 +2435,18 @@ // TODO: we could skip this if !hasTerms; but // that's rare so won't help much // metadata - numBytes = in.readVInt(); + if (postingsReader.pulsed()) { + int start = (int)in.getFilePointer(); + numBytes = in.readVInt(); + in.seek(in.getFilePointer() + numBytes); + numBytes = in.readVInt(); + numBytes = numBytes - start + (int)in.getFilePointer(); + in.seek(start); + } else { + numBytes = in.readVInt(); + } if (bytes == null) { bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; bytesReader = new ByteArrayDataInput(); @@ -2431,6 +2454,7 @@ bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; } in.readBytes(bytes, 0, numBytes); bytesReader.reset(bytes, 0, numBytes); Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (revision 1517792) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (working copy) @@ -856,11 +856,13 @@ if (isLeafBlock) { subIndices = null; + termCount = length; for (PendingEntry ent : slice) { assert ent.isTerm; PendingTerm term = (PendingTerm) ent; BlockTermState state = term.state; final int suffix = term.term.length - prefixLength; + state.termBlockOrd = termCount; // if (DEBUG) { // BytesRef suffixBytes = new BytesRef(suffix); // System.arraycopy(term.term.bytes, prefixLength, suffixBytes.bytes, 0, suffix); @@ -888,14 +890,19 @@ bytesWriter.reset(); absolute = false; } - termCount = length; } else { subIndices = new ArrayList>(); termCount = 0; for (PendingEntry ent : slice) { if (ent.isTerm) { + termCount++; + } + } + for (PendingEntry ent : slice) { + if (ent.isTerm) { PendingTerm term = (PendingTerm) ent; BlockTermState state = term.state; + state.termBlockOrd = termCount; final int suffix = term.term.length - prefixLength; // if (DEBUG) { // BytesRef suffixBytes = new BytesRef(suffix); @@ -932,8 +939,6 @@ bytesWriter.writeTo(metaWriter); bytesWriter.reset(); absolute = false; - - termCount++; } else { PendingBlock block = (PendingBlock) ent; final int suffix = block.prefix.length - prefixLength; @@ -976,7 +981,12 @@ statsWriter.reset(); // Write term meta data byte[] blob - out.writeVInt((int) metaWriter.getFilePointer()); + if (!postingsWriter.pulsed()) { + out.writeVInt((int) metaWriter.getFilePointer()); + } else if (metaWriter.getFilePointer() == 0) { + out.writeVInt(0); + out.writeVInt(0); + } metaWriter.writeTo(out); metaWriter.reset(); Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (revision 1517792) +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (working copy) @@ -69,4 +69,8 @@ @Override public abstract void close() throws IOException; + + public boolean pulsed() { + return false; + } } Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java (revision 1517792) +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java (working copy) @@ -77,4 +77,8 @@ @Override public abstract void close() throws IOException; + + public boolean pulsed() { + return false; + } } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (revision 1517792) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (working copy) @@ -234,7 +234,6 @@ } else { termState.singletonDocID = -1; termState.docStartFP += in.readVLong(); - System.out.println(termState.docStartFP); } if (fieldHasPositions) { termState.posStartFP += in.readVLong();