Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java (revision 1334418) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java (working copy) @@ -111,25 +111,26 @@ @Override protected int readSkipData(int level, IndexInput skipStream) throws IOException { int delta; - if (currentFieldStoresPayloads) { - // the current field stores payloads. + if (currentFieldStoresPayloads || currentFieldStoresOffsets) { + // the current field stores payloads and/or offsets. // if the doc delta is odd then we have - // to read the current payload length - // because it differs from the length of the - // previous payload + // to read the current payload/offset lengths + // because it differs from the lengths of the + // previous payload/offset delta = skipStream.readVInt(); if ((delta & 1) != 0) { - payloadLength[level] = skipStream.readVInt(); + if (currentFieldStoresPayloads) { + payloadLength[level] = skipStream.readVInt(); + } + if (currentFieldStoresOffsets) { + offsetLength[level] = skipStream.readVInt(); + } } delta >>>= 1; } else { delta = skipStream.readVInt(); } - if (currentFieldStoresOffsets) { - offsetLength[level] = skipStream.readVInt(); - } - freqPointer[level] += skipStream.readVInt(); proxPointer[level] += skipStream.readVInt(); Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java (revision 1334418) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListWriter.java (working copy) @@ -32,6 +32,7 @@ public class Lucene40SkipListWriter extends MultiLevelSkipListWriter { private int[] lastSkipDoc; private int[] lastSkipPayloadLength; + private int[] lastSkipOffsetLength; private long[] lastSkipFreqPointer; private long[] lastSkipProxPointer; @@ -53,6 +54,7 @@ lastSkipDoc = new int[numberOfSkipLevels]; lastSkipPayloadLength = new int[numberOfSkipLevels]; + lastSkipOffsetLength = new int[numberOfSkipLevels]; lastSkipFreqPointer = new long[numberOfSkipLevels]; lastSkipProxPointer = new long[numberOfSkipLevels]; } @@ -61,6 +63,8 @@ * Sets the values for the current skip data. */ public void setSkipData(int doc, boolean storePayloads, int payloadLength, boolean storeOffsets, int offsetLength) { + assert storePayloads || payloadLength == -1; + assert storeOffsets || offsetLength == -1; this.curDoc = doc; this.curStorePayloads = storePayloads; this.curPayloadLength = payloadLength; @@ -76,6 +80,7 @@ super.resetSkip(); Arrays.fill(lastSkipDoc, 0); Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list + Arrays.fill(lastSkipOffsetLength, -1); // we don't have to write the first length in the skip list Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); if (proxOutput != null) Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); @@ -83,49 +88,55 @@ @Override protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { - // To efficiently store payloads in the posting lists we do not store the length of - // every payload. Instead we omit the length for a payload if the previous payload had - // the same length. - // However, in order to support skipping the payload length at every skip point must be known. + // To efficiently store payloads/offsets in the posting lists we do not store the length of + // every payload/offset. Instead we omit the length if the previous lengths were the same + // + // However, in order to support skipping, the length at every skip point must be known. // So we use the same length encoding that we use for the posting lists for the skip data as well: - // Case 1: current field does not store payloads + // Case 1: current field does not store payloads/offsets // SkipDatum --> DocSkip, FreqSkip, ProxSkip // DocSkip,FreqSkip,ProxSkip --> VInt // DocSkip records the document number before every SkipInterval th document in TermFreqs. // Document numbers are represented as differences from the previous value in the sequence. - // Case 2: current field stores payloads - // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip + // Case 2: current field stores payloads/offsets + // SkipDatum --> DocSkip, PayloadLength?,OffsetLength?,FreqSkip,ProxSkip // DocSkip,FreqSkip,ProxSkip --> VInt - // PayloadLength --> VInt + // PayloadLength,OffsetLength--> VInt // In this case DocSkip/2 is the difference between // the current and the previous value. If DocSkip // is odd, then a PayloadLength encoded as VInt follows, // if DocSkip is even, then it is assumed that the - // current payload length equals the length at the previous + // current payload/offset lengths equals the lengths at the previous // skip point - if (curStorePayloads) { - int delta = curDoc - lastSkipDoc[level]; - if (curPayloadLength == lastSkipPayloadLength[level]) { - // the current payload length equals the length at the previous skip point, - // so we don't store the length again - skipBuffer.writeVInt(delta * 2); + int delta = curDoc - lastSkipDoc[level]; + + if (curStorePayloads || curStoreOffsets) { + assert curStorePayloads || curPayloadLength == lastSkipPayloadLength[level]; + assert curStoreOffsets || curOffsetLength == lastSkipOffsetLength[level]; + + if (curPayloadLength == lastSkipPayloadLength[level] && curOffsetLength == lastSkipOffsetLength[level]) { + // the current payload/offset lengths equals the lengths at the previous skip point, + // so we don't store the lengths again + skipBuffer.writeVInt(delta << 1); } else { - // the payload length is different from the previous one. We shift the DocSkip, - // set the lowest bit and store the current payload length as VInt. - skipBuffer.writeVInt(delta * 2 + 1); - skipBuffer.writeVInt(curPayloadLength); - lastSkipPayloadLength[level] = curPayloadLength; + // the payload and/or offset length is different from the previous one. We shift the DocSkip, + // set the lowest bit and store the current payload and/or offset lengths as VInts. + skipBuffer.writeVInt(delta << 1 | 1); + + if (curStorePayloads) { + skipBuffer.writeVInt(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; + } + if (curStoreOffsets) { + skipBuffer.writeVInt(curOffsetLength); + lastSkipOffsetLength[level] = curOffsetLength; + } } } else { - // current field does not store payloads - skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + // current field does not store payloads or offsets + skipBuffer.writeVInt(delta); } - // TODO: not sure it really helps to shove this somewhere else if its the same as the last skip - if (curStoreOffsets) { - skipBuffer.writeVInt(curOffsetLength); - } - skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java (revision 1334418) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java (working copy) @@ -138,11 +138,11 @@ //if (DEBUG) System.out.println("SPW: startTerm freqOut.fp=" + freqStart); if (proxOut != null) { proxStart = proxOut.getFilePointer(); - // force first payload to write its length - lastPayloadLength = -1; - // force first offset to write its length - lastOffsetLength = -1; } + // force first payload to write its length + lastPayloadLength = -1; + // force first offset to write its length + lastOffsetLength = -1; skipListWriter.resetSkip(); }