Index: lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java (revision 1370367) +++ lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java (working copy) @@ -152,6 +152,10 @@ return lastPayloadByteUpto; } + public int getNextSkipDoc() { + return skipDoc[0]; + } + @Override protected void seekChild(int level) throws IOException { super.seekChild(level); Index: lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java (revision 1370367) +++ lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java (working copy) @@ -355,6 +355,10 @@ // no skip data for this term): private int skipOffset; + // docID for next skip point, we won't use skipper if + // target docID is not larger than this + private int nextSkipDoc; + private Bits liveDocs; public BlockDocsEnum(FieldInfo fieldInfo) throws IOException { @@ -391,6 +395,7 @@ } accum = 0; docUpto = 0; + nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block docBufferUpto = BLOCK_SIZE; skipped = false; return this; @@ -475,11 +480,14 @@ @Override public int advance(int target) throws IOException { // nocommit make frq block load lazy/skippable + if (DEBUG) { + System.out.println(" FPR.advance target=" + target); + } - // nocommit use skipper!!! it has next last doc id!! + // current skip docID < docIDs generated from current buffer <= next skip docID + // we don't need to skip if target is buffered already + if (docFreq > BLOCK_SIZE && target > nextSkipDoc) { - if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) { - if (DEBUG) { System.out.println("load skipper"); } @@ -502,22 +510,29 @@ skipped = true; } - final int newDocUpto = skipper.skipTo(target); + final int newDocUpto = skipper.skipTo(target) + 1; if (newDocUpto > docUpto) { // Skipper moved if (DEBUG) { System.out.println("skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer()); } - assert newDocUpto % BLOCK_SIZE == (BLOCK_SIZE-1): "got " + newDocUpto; - docUpto = newDocUpto+1; + assert newDocUpto % BLOCK_SIZE == 0: "got " + newDocUpto; + docUpto = newDocUpto; // Force to read next block docBufferUpto = BLOCK_SIZE; accum = skipper.getDoc(); // actually, this is just lastSkipEntry docIn.seek(skipper.getDocPointer()); // now point to the block we want to search - } + } + nextSkipDoc = skipper.getNextSkipDoc(); } + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } + if (docBufferUpto == BLOCK_SIZE) { + refillDocs(); + } // Now scan... this is an inlined/pared down version // of nextDoc(): @@ -525,18 +540,6 @@ if (DEBUG) { System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - - // nocommit: in theory we should not hit this? ie - // skipper should already have moved us to the block - // containing the doc? yet assert false trips ... i - // think because if you advance w/o having done a - // nextDoc yet()... can we assert/remove this? - if (docBufferUpto == BLOCK_SIZE) { - refillDocs(); - } accum += docDeltaBuffer[docBufferUpto]; docUpto++; @@ -544,6 +547,10 @@ break; } docBufferUpto++; + + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } } if (liveDocs == null || liveDocs.get(accum)) { @@ -622,6 +629,8 @@ // no skip data for this term): private int skipOffset; + private int nextSkipDoc; + private Bits liveDocs; public BlockDocsAndPositionsEnum(FieldInfo fieldInfo) throws IOException { @@ -664,6 +673,7 @@ doc = -1; accum = 0; docUpto = 0; + nextSkipDoc = BLOCK_SIZE - 1; docBufferUpto = BLOCK_SIZE; skipped = false; return this; @@ -683,6 +693,7 @@ //System.out.println("["+docFreq+"]"+" refillDoc"); final int left = docFreq - docUpto; assert left > 0; + if (left >= BLOCK_SIZE) { if (DEBUG) { System.out.println(" fill doc block from fp=" + docIn.getFilePointer()); @@ -785,9 +796,8 @@ // nocommit 2 is heuristic guess!! // nocommit put cheating back! does it help? - // nocommit use skipper!!! it has next last doc id!! //if (docFreq > BLOCK_SIZE && target - (BLOCK_SIZE - docBufferUpto) - 2*BLOCK_SIZE > accum) { - if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) { + if (docFreq > BLOCK_SIZE && target > nextSkipDoc) { if (DEBUG) { System.out.println(" try skipper"); } @@ -815,7 +825,7 @@ skipped = true; } - final int newDocUpto = skipper.skipTo(target); + final int newDocUpto = skipper.skipTo(target) + 1; if (newDocUpto > docUpto) { // Skipper moved @@ -823,8 +833,8 @@ System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto()); } - assert newDocUpto % BLOCK_SIZE == (BLOCK_SIZE-1): "got " + newDocUpto; - docUpto = newDocUpto+1; + assert newDocUpto % BLOCK_SIZE == 0: "got " + newDocUpto; + docUpto = newDocUpto; // Force to read next block docBufferUpto = BLOCK_SIZE; @@ -833,7 +843,14 @@ posPendingFP = skipper.getPosPointer(); posPendingCount = skipper.getPosBufferUpto(); } + nextSkipDoc = skipper.getNextSkipDoc(); + } + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; } + if (docBufferUpto == BLOCK_SIZE) { + refillDocs(); + } // Now scan... this is an inlined/pared down version // of nextDoc(): @@ -841,19 +858,6 @@ if (DEBUG) { System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); } - if (docUpto == docFreq) { - return doc = NO_MORE_DOCS; - } - // nocommit: in theory we should not hit this? ie - // skipper should already have moved us to the block - // containing the doc? yet assert false trips ... i - // think because if you advance w/o having done a - // nextDoc yet()... can we assert/remove this? - if (docBufferUpto == BLOCK_SIZE) { - // nocommit hmm skip freq? but: we don't ever - // scan over more than one block? - refillDocs(); - } accum += docDeltaBuffer[docBufferUpto]; freq = freqBuffer[docBufferUpto]; posPendingCount += freq; @@ -863,6 +867,9 @@ if (accum >= target) { break; } + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } } if (liveDocs == null || liveDocs.get(accum)) { @@ -1047,6 +1054,8 @@ // no skip data for this term): private int skipOffset; + private int nextSkipDoc; + private Bits liveDocs; public EverythingEnum(FieldInfo fieldInfo) throws IOException { @@ -1110,6 +1119,7 @@ doc = -1; accum = 0; docUpto = 0; + nextSkipDoc = BLOCK_SIZE - 1; docBufferUpto = BLOCK_SIZE; skipped = false; return this; @@ -1280,9 +1290,8 @@ // nocommit 2 is heuristic guess!! // nocommit put cheating back! does it help? - // nocommit use skipper!!! it has next last doc id!! //if (docFreq > BLOCK_SIZE && target - (BLOCK_SIZE - docBufferUpto) - 2*BLOCK_SIZE > accum) { - if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) { + if (docFreq > BLOCK_SIZE && target > nextSkipDoc) { if (DEBUG) { System.out.println(" try skipper"); @@ -1312,15 +1321,15 @@ skipped = true; } - final int newDocUpto = skipper.skipTo(target); + final int newDocUpto = skipper.skipTo(target) + 1; if (newDocUpto > docUpto) { // Skipper moved if (DEBUG) { System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset); } - assert newDocUpto % BLOCK_SIZE == (BLOCK_SIZE-1): "got " + newDocUpto; - docUpto = newDocUpto+1; + assert newDocUpto % BLOCK_SIZE == 0: "got " + newDocUpto; + docUpto = newDocUpto; // Force to read next block docBufferUpto = BLOCK_SIZE; @@ -1332,25 +1341,52 @@ lastStartOffset = skipper.getStartOffset(); payloadByteUpto = skipper.getPayloadByteUpto(); } + nextSkipDoc = skipper.getNextSkipDoc(); } + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } + if (docBufferUpto == BLOCK_SIZE) { + refillDocs(); + } - // nocommit inline nextDoc here - // Now scan: - while (nextDoc() != NO_MORE_DOCS) { - if (doc >= target) { - if (DEBUG) { - System.out.println(" advance return doc=" + doc); - } - return doc; + while (true) { + if (DEBUG) { + System.out.println(" scan doc=" + accum + " docBufferUpto=" + docBufferUpto); } + accum += docDeltaBuffer[docBufferUpto]; + freq = freqBuffer[docBufferUpto]; + posPendingCount += freq; + docBufferUpto++; + docUpto++; + + if (accum >= target) { + break; + } + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } } - if (DEBUG) { - System.out.println(" advance return doc=END"); + if (liveDocs == null || liveDocs.get(accum)) { + if (DEBUG) { + System.out.println(" return doc=" + accum); + } + if (indexHasPayloads) { + payloadByteUpto += payloadLength; + payloadLength = 0; + } + position = 0; + payloadLength = 0; + lastStartOffset = 0; + return doc = accum; + } else { + if (DEBUG) { + System.out.println(" now do nextDoc()"); + } + return nextDoc(); } - - return NO_MORE_DOCS; } // nocommit in theory we could avoid loading frq block Index: lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java (revision 1370367) +++ lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java (working copy) @@ -37,7 +37,7 @@ * block, only record skip data at the start its start point(if it exist). * * For each skip point, we will record: - * 1. lastDocID, + * 1. docID in former position, i.e. docID[11] for position 12,etc. * 2. its related file points(position, payload), * 3. related numbers or uptos(position, payload). * 4. start offset. Index: lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java (revision 1370367) +++ lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java (working copy) @@ -59,7 +59,7 @@ private int skipInterval[]; // skipInterval of each level private int[] numSkipped; // number of docs skipped per level - private int[] skipDoc; // doc id of current skip entry per level + protected int[] skipDoc; // doc id of current skip entry per level private int lastDoc; // doc id of last read skip entry with docId <= target private long[] childPointer; // child pointer of current skip entry per level private long lastChildPointer; // childPointer of last read skip entry with docId <= target