Index: lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (revision 1221659) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (working copy) @@ -232,49 +232,54 @@ return false; } + private boolean canReuse(DocsAndPositionsEnum reuse, Bits liveDocs, boolean payloads) { + if (reuse != null && (reuse instanceof SegmentDocsAndPositionsEnumBase)) { + SegmentDocsAndPositionsEnumBase docsEnum = (SegmentDocsAndPositionsEnumBase) reuse; + // If you are using ParellelReader, and pass in a + // reused DocsEnum, it could have come from another + // reader also using standard codec + if (docsEnum.startFreqIn == freqIn && liveDocs == docsEnum.liveDocs) { + // we only reuse if the the actual the incoming enum has the same liveDocs as the given liveDocs + + return payloads ? reuse instanceof SegmentDocsAndPositionsAndPayloadsEnum : !(reuse instanceof SegmentDocsAndPositionsAndPayloadsEnum); + } + } + return false; + } + private DocsEnum newDocsEnum(Bits liveDocs, FieldInfo fieldInfo, StandardTermState termState) throws IOException { if (liveDocs == null) { - return new AllDocsSegmentDocsEnum(freqIn).reset(fieldInfo, termState); + return new SegmentAllDocDocsEnum(freqIn).reset(fieldInfo, termState); } else { - return new LiveDocsSegmentDocsEnum(freqIn, liveDocs).reset(fieldInfo, termState); + return new SegmentLiveDocsDocsEnum(freqIn, liveDocs).reset(fieldInfo, termState); } } + + private DocsAndPositionsEnum newDocsAndPosEnum(Bits liveDocs, FieldInfo fieldInfo, StandardTermState termState) throws IOException { + if (fieldInfo.storePayloads) { + if (liveDocs == null) { + return new SegmentAllDocsAndPositionsAndPayloadsEnum(freqIn, proxIn).reset(fieldInfo, termState); + } else { + return new SegmentLiveDocsAndPositionsAndPayloadsEnum(freqIn, proxIn, liveDocs).reset(fieldInfo, termState); + } + } else if (liveDocs == null) { + return new SegmentAllDocsAndPositionsEnum(freqIn, proxIn).reset(fieldInfo, termState); + } else { + return new SegmentLiveDocsAndPositionsEnum(freqIn, proxIn, liveDocs).reset(fieldInfo, termState); + } + } @Override public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { if (fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { return null; } - - // TODO: refactor - if (fieldInfo.storePayloads) { - SegmentDocsAndPositionsAndPayloadsEnum docsEnum; - if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsAndPayloadsEnum)) { - docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); - } else { - docsEnum = (SegmentDocsAndPositionsAndPayloadsEnum) reuse; - if (docsEnum.startFreqIn != freqIn) { - // If you are using ParellelReader, and pass in a - // reused DocsEnum, it could have come from another - // reader also using standard codec - docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); - } - } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); + SegmentDocsAndPositionsEnumBase docsEnum; + if (canReuse(reuse, liveDocs, fieldInfo.storePayloads)) { + docsEnum = (SegmentDocsAndPositionsEnumBase) reuse; + return docsEnum.reset(fieldInfo,(StandardTermState) termState); } else { - SegmentDocsAndPositionsEnum docsEnum; - if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsEnum)) { - docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); - } else { - docsEnum = (SegmentDocsAndPositionsEnum) reuse; - if (docsEnum.startFreqIn != freqIn) { - // If you are using ParellelReader, and pass in a - // reused DocsEnum, it could have come from another - // reader also using standard codec - docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); - } - } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); + return newDocsAndPosEnum(liveDocs, fieldInfo, (StandardTermState) termState); } } @@ -285,9 +290,9 @@ protected final int[] docs = new int[BUFFERSIZE]; protected final int[] freqs = new int[BUFFERSIZE]; - final IndexInput freqIn; // reuse + protected final IndexInput freqIn; // reuse final IndexInput startFreqIn; // reuse - Lucene40SkipListReader skipper; // reuse - lazy loaded + protected Lucene40SkipListReader skipper; // reuse - lazy loaded protected boolean indexOmitsTF; // does current field omit term freq? protected boolean storePayloads; // does current field store payloads? @@ -486,9 +491,9 @@ } } - private final class AllDocsSegmentDocsEnum extends SegmentDocsEnumBase { + private final class SegmentAllDocDocsEnum extends SegmentDocsEnumBase { - AllDocsSegmentDocsEnum(IndexInput startFreqIn) throws IOException { + SegmentAllDocDocsEnum(IndexInput startFreqIn) throws IOException { super(startFreqIn, null); assert liveDocs == null; } @@ -563,9 +568,9 @@ } - private final class LiveDocsSegmentDocsEnum extends SegmentDocsEnumBase { + private final class SegmentLiveDocsDocsEnum extends SegmentDocsEnumBase { - LiveDocsSegmentDocsEnum(IndexInput startFreqIn, Bits liveDocs) throws IOException { + SegmentLiveDocsDocsEnum(IndexInput startFreqIn, Bits liveDocs) throws IOException { super(startFreqIn, liveDocs); assert liveDocs != null; } @@ -659,44 +664,42 @@ } } - // TODO specialize DocsAndPosEnum too - // Decodes docs & positions. payloads are not present. - private final class SegmentDocsAndPositionsEnum extends DocsAndPositionsEnum { + private abstract class SegmentDocsAndPositionsEnumBase extends DocsAndPositionsEnum { final IndexInput startFreqIn; - private final IndexInput freqIn; - private final IndexInput proxIn; - int limit; // number of docs in this posting - int ord; // how many docs we've read - int doc = -1; // doc we last read - int accum; // accumulator for doc deltas - int freq; // freq we last read - int position; + protected final IndexInput freqIn; + protected final IndexInput proxIn; + protected int limit; // number of docs in this posting + protected int ord; // how many docs we've read + protected int doc = -1; // doc we last read + protected int accum; // accumulator for doc deltas + protected int freq; // freq we last read + protected int position; - Bits liveDocs; + protected final Bits liveDocs; - long freqOffset; - int skipOffset; - long proxOffset; + protected long freqOffset; + protected int skipOffset; + protected long proxOffset; - int posPendingCount; + protected int posPendingCount; - boolean skipped; - Lucene40SkipListReader skipper; - private long lazyProxPointer; - - public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + protected boolean skipped; + protected Lucene40SkipListReader skipper; + protected long lazyProxPointer; + private boolean storePayloads; + + SegmentDocsAndPositionsEnumBase(IndexInput freqIn, IndexInput proxIn, Bits liveDocs, boolean storePayloads) throws IOException { startFreqIn = freqIn; this.freqIn = (IndexInput) freqIn.clone(); this.proxIn = (IndexInput) proxIn.clone(); + this.liveDocs = liveDocs; + this.storePayloads = storePayloads; } - - public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { + + SegmentDocsAndPositionsEnumBase reset(FieldInfo fieldInfo, StandardTermState termState) throws IOException { assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - assert !fieldInfo.storePayloads; - this.liveDocs = liveDocs; - // TODO: for full enum case (eg segment merging) this // seek is unnecessary; maybe we can avoid in such // cases @@ -721,60 +724,59 @@ return this; } + + @Override + public final int docID() { + return doc; + } @Override + public final int freq() { + return freq; + } + + @Override + public BytesRef getPayload() throws IOException { + throw new IOException("No payloads exist for this field!"); + } + + @Override + public boolean hasPayload() { + return false; + } + + @Override public int nextDoc() throws IOException { // if (DEBUG) System.out.println("SPR.nextDoc seg=" + segment + " freqIn.fp=" + freqIn.getFilePointer()); - while(true) { - if (ord == limit) { - // if (DEBUG) System.out.println(" return END"); - return doc = NO_MORE_DOCS; - } - - ord++; - - // Decode next doc/freq pair - final int code = freqIn.readVInt(); - - accum += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - posPendingCount += freq; - - if (liveDocs == null || liveDocs.get(accum)) { - break; - } + if (ord == limit) { + // if (DEBUG) System.out.println(" return END"); + return doc = NO_MORE_DOCS; } + return doc = readNextDoc(); + } + protected final int readNextDoc() throws IOException { + ord++; + // Decode next doc/freq pair + final int code = freqIn.readVInt(); + accum += code >>> 1; // shift off low bit + if ((code & 1) != 0) { // if low bit is set + freq = 1; // freq is one + } else { + freq = freqIn.readVInt(); // else read freq + } + posPendingCount += freq; position = 0; - // if (DEBUG) System.out.println(" return doc=" + doc); - return (doc = accum); + return accum; } @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - return freq; - } - - @Override public int advance(int target) throws IOException { - //System.out.println("StandardR.D&PE advance target=" + target); - if ((target - skipInterval) >= doc && limit >= skipMinimum) { - // There are enough docs in the posting to have // skip data, and it isn't too close - if (skipper == null) { // This is the first time this enum has ever been used for skipping -- do lazy init skipper = new Lucene40SkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); @@ -788,21 +790,14 @@ skipper.init(freqOffset+skipOffset, freqOffset, proxOffset, - limit, false); + limit, storePayloads); skipped = true; } final int newOrd = skipper.skipTo(target); - if (newOrd > ord) { - // Skipper moved - ord = newOrd; - doc = accum = skipper.getDoc(); - freqIn.seek(skipper.getFreqPointer()); - lazyProxPointer = skipper.getProxPointer(); - posPendingCount = 0; - position = 0; + doAfterSkipperMoved(newOrd); } } @@ -813,209 +808,109 @@ return doc; } - + + protected void doAfterSkipperMoved(int newOrd) throws IOException { + // Skipper moved + ord = newOrd; + doc = accum = skipper.getDoc(); + freqIn.seek(skipper.getFreqPointer()); + lazyProxPointer = skipper.getProxPointer(); + posPendingCount = 0; + position = 0; + } + @Override public int nextPosition() throws IOException { - if (lazyProxPointer != -1) { proxIn.seek(lazyProxPointer); lazyProxPointer = -1; } // scan over any docs that were iterated without their positions - if (posPendingCount > freq) { + int posToSkip = posPendingCount; + final int frq = freq; + final IndexInput prx = proxIn; + if (posToSkip > frq) { position = 0; - while(posPendingCount != freq) { - if ((proxIn.readByte() & 0x80) == 0) { - posPendingCount--; + while(posToSkip != frq) { + if ((prx.readByte() & 0x80) == 0) { + posToSkip--; } } } position += proxIn.readVInt(); - posPendingCount--; + posPendingCount = posToSkip-1; assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount; return position; } - /** Returns the payload at this position, or null if no - * payload was indexed. */ - @Override - public BytesRef getPayload() throws IOException { - throw new IOException("No payloads exist for this field!"); + } + + private final class SegmentAllDocsAndPositionsEnum extends SegmentDocsAndPositionsEnumBase { + + SegmentAllDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + super(freqIn, proxIn, null, false); } - @Override - public boolean hasPayload() { - return false; + SegmentDocsAndPositionsEnumBase reset(FieldInfo fieldInfo, StandardTermState termState) throws IOException { + assert !fieldInfo.storePayloads; + return super.reset(fieldInfo, termState); } } - // Decodes docs & positions & payloads - private class SegmentDocsAndPositionsAndPayloadsEnum extends DocsAndPositionsEnum { - final IndexInput startFreqIn; - private final IndexInput freqIn; - private final IndexInput proxIn; + private final class SegmentLiveDocsAndPositionsEnum extends SegmentDocsAndPositionsEnumBase { - int limit; // number of docs in this posting - int ord; // how many docs we've read - int doc = -1; // doc we last read - int accum; // accumulator for doc deltas - int freq; // freq we last read - int position; - - Bits liveDocs; - - long freqOffset; - int skipOffset; - long proxOffset; - - int posPendingCount; - int payloadLength; - boolean payloadPending; - - boolean skipped; - Lucene40SkipListReader skipper; - private BytesRef payload; - private long lazyProxPointer; - - public SegmentDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { - startFreqIn = freqIn; - this.freqIn = (IndexInput) freqIn.clone(); - this.proxIn = (IndexInput) proxIn.clone(); + SegmentLiveDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn, Bits liveDocs) throws IOException { + super(freqIn, proxIn, liveDocs, false); } - - public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { - assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - assert fieldInfo.storePayloads; - if (payload == null) { - payload = new BytesRef(); - payload.bytes = new byte[1]; - } - - this.liveDocs = liveDocs; - - // TODO: for full enum case (eg segment merging) this - // seek is unnecessary; maybe we can avoid in such - // cases - freqIn.seek(termState.freqOffset); - lazyProxPointer = termState.proxOffset; - - limit = termState.docFreq; - ord = 0; - doc = -1; - accum = 0; - position = 0; - - skipped = false; - posPendingCount = 0; - payloadPending = false; - - freqOffset = termState.freqOffset; - proxOffset = termState.proxOffset; - skipOffset = termState.skipOffset; - //System.out.println("StandardR.D&PE reset seg=" + segment + " limit=" + limit + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " this=" + this); - - return this; + + SegmentDocsAndPositionsEnumBase reset(FieldInfo fieldInfo, StandardTermState termState) throws IOException { + assert !fieldInfo.storePayloads; + return super.reset(fieldInfo, termState); } - + @Override - public int nextDoc() throws IOException { - while(true) { - if (ord == limit) { - //System.out.println("StandardR.D&PE seg=" + segment + " nextDoc return doc=END"); - return doc = NO_MORE_DOCS; - } - - ord++; - - // Decode next doc/freq pair - final int code = freqIn.readVInt(); - - accum += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - posPendingCount += freq; - - if (liveDocs == null || liveDocs.get(accum)) { + public final int nextDoc() throws IOException { + if (ord == limit) { + // if (DEBUG) System.out.println(" return END"); + return doc = NO_MORE_DOCS; + } + int currentDoc; + for (int i = ord; i < limit; i++) { + currentDoc = readNextDoc(); + if (currentDoc == NO_MORE_DOCS) { break; } + if (liveDocs.get(currentDoc)) { + return doc = currentDoc; + } } - - position = 0; - - //System.out.println("StandardR.D&PE nextDoc seg=" + segment + " return doc=" + doc); - return (doc = accum); + return doc = NO_MORE_DOCS; } + } + + // Decodes docs & positions & payloads + private abstract class SegmentDocsAndPositionsAndPayloadsEnum extends SegmentDocsAndPositionsEnumBase { + private int payloadLength; + private boolean payloadPending; + private BytesRef payload; - @Override - public int docID() { - return doc; + + SegmentDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn, Bits liveDocs) throws IOException { + super(freqIn, proxIn, liveDocs, true); } - - @Override - public int freq() { - return freq; + + protected final void doAfterSkipperMoved(int newOrd) throws IOException { + super.doAfterSkipperMoved(newOrd); + payloadPending = false; + payloadLength = skipper.getPayloadLength(); } - + @Override - public int advance(int target) throws IOException { - - //System.out.println("StandardR.D&PE advance seg=" + segment + " target=" + target + " this=" + this); - - if ((target - skipInterval) >= doc && limit >= skipMinimum) { - - // There are enough docs in the posting to have - // skip data, and it isn't too close - - if (skipper == null) { - // This is the first time this enum has ever been used for skipping -- do lazy init - skipper = new Lucene40SkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); - } - - if (!skipped) { - - // This is the first time this posting has - // skipped, since reset() was called, so now we - // load the skip data for this posting - //System.out.println(" init skipper freqOffset=" + freqOffset + " skipOffset=" + skipOffset + " vs len=" + freqIn.length()); - skipper.init(freqOffset+skipOffset, - freqOffset, proxOffset, - limit, true); - - skipped = true; - } - - final int newOrd = skipper.skipTo(target); - - if (newOrd > ord) { - // Skipper moved - ord = newOrd; - doc = accum = skipper.getDoc(); - freqIn.seek(skipper.getFreqPointer()); - lazyProxPointer = skipper.getProxPointer(); - posPendingCount = 0; - position = 0; - payloadPending = false; - payloadLength = skipper.getPayloadLength(); - } - } - - // Now, linear scan for the rest: - do { - nextDoc(); - } while (target > doc); - - return doc; - } - - @Override public int nextPosition() throws IOException { if (lazyProxPointer != -1) { @@ -1029,26 +924,8 @@ payloadPending = false; } - // scan over any docs that were iterated without their positions - while(posPendingCount > freq) { + scanPendingPositions(); - final int code = proxIn.readVInt(); - - if ((code & 1) != 0) { - // new payload length - payloadLength = proxIn.readVInt(); - assert payloadLength >= 0; - } - - assert payloadLength != -1; - proxIn.seek(proxIn.getFilePointer() + payloadLength); - - posPendingCount--; - position = 0; - payloadPending = false; - //System.out.println("StandardR.D&PE skipPos"); - } - // read next position if (payloadPending && payloadLength > 0) { // payload wasn't retrieved for last position @@ -1073,6 +950,31 @@ //System.out.println("StandardR.D&PE nextPos return pos=" + position); return position; } + + protected final void scanPendingPositions() throws IOException { + // scan over any docs that were iterated without their positions + final int pending = posPendingCount - freq; + if (pending > 0) { + final IndexInput prx = proxIn; + int payloadLength = this.payloadLength; + + for (int i = 0; i < pending; i++) { + final int code = prx.readVInt(); + + if ((code & 1) != 0) { + // new payload length + payloadLength = prx.readVInt(); + assert payloadLength >= 0; + } + assert payloadLength != -1; + prx.seek(prx.getFilePointer() + payloadLength); + } + posPendingCount = freq; + position = 0; + this.payloadLength = payloadLength; + payloadPending = false; + } + } /** Returns the payload at this position, or null if no * payload was indexed. */ @@ -1098,5 +1000,51 @@ public boolean hasPayload() { return payloadPending && payloadLength > 0; } + + public SegmentDocsAndPositionsEnumBase reset(FieldInfo fieldInfo, StandardTermState termState) throws IOException { + assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + assert fieldInfo.storePayloads; + if (payload == null) { + payload = new BytesRef(); + payload.bytes = new byte[1]; + } + payloadPending = false; + return super.reset(fieldInfo, termState); + } + } + + private final class SegmentAllDocsAndPositionsAndPayloadsEnum extends SegmentDocsAndPositionsAndPayloadsEnum { + + SegmentAllDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + super(freqIn, proxIn, null); + } + + } + + private final class SegmentLiveDocsAndPositionsAndPayloadsEnum extends SegmentDocsAndPositionsAndPayloadsEnum { + + SegmentLiveDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn, Bits liveDocs) throws IOException { + super(freqIn, proxIn, liveDocs); + } + + @Override + public final int nextDoc() throws IOException { + if (ord == limit) { + // if (DEBUG) System.out.println(" return END"); + return doc = NO_MORE_DOCS; + } + int currentDoc; + for (int i = ord; i < limit; i++) { + currentDoc = readNextDoc(); + if (currentDoc == NO_MORE_DOCS) { + doc = NO_MORE_DOCS; + } + if (liveDocs.get(currentDoc)) { + return doc = currentDoc; + } + } + return doc = NO_MORE_DOCS; + } + } }