Index: lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (révision 1435301) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (copie de travail) @@ -17,22 +17,39 @@ * limitations under the License. */ +import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.CannedTokenStream; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import com.carrotsearch.randomizedtesting.generators.RandomInts; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + public class TestPayloadsOnVectors extends LuceneTestCase { /** some docs have payload att, some not */ @@ -141,4 +158,314 @@ writer.close(); dir.close(); } + + // custom impl to test cases that are forbidden by the default OffsetAttribute impl + static class PermissiveOffsetAttributeImpl extends AttributeImpl implements OffsetAttribute { + + int start, end; + + @Override + public int startOffset() { + return start; + } + + @Override + public int endOffset() { + return end; + } + + @Override + public void setOffset(int startOffset, int endOffset) { + // no check! + start = startOffset; + end = endOffset; + } + + @Override + public void clear() { + start = end = 0; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (other instanceof PermissiveOffsetAttributeImpl) { + PermissiveOffsetAttributeImpl o = (PermissiveOffsetAttributeImpl) other; + return o.start == start && o.end == end; + } + + return false; + } + + @Override + public int hashCode() { + return start + 31 * end; + } + + @Override + public void copyTo(AttributeImpl target) { + OffsetAttribute t = (OffsetAttribute) target; + t.setOffset(start, end); + } + + } + + static BytesRef randomPayload() { + final int len = random().nextInt(5); + if (len == 0) { + return null; + } + final BytesRef payload = new BytesRef(len); + random().nextBytes(payload.bytes); + payload.length = len; + return payload; + } + + class RandomTokenStream extends TokenStream { + + final String[] terms; + final int[] positionsIncrements; + final int[] positions; + final int[] startOffsets, endOffsets; + final BytesRef[] payloads; + + final Map> positionToTerms; + final Map> startOffsetToTerms; + + final CharTermAttribute termAtt; + final PositionIncrementAttribute piAtt; + final OffsetAttribute oAtt; + final PayloadAttribute pAtt; + int i = 0; + + RandomTokenStream(int len, String[] sampleTerms, boolean weird) { + terms = new String[len]; + positionsIncrements = new int[len]; + positions = new int[len]; + startOffsets = new int[len]; + endOffsets = new int[len]; + payloads = new BytesRef[len]; + for (int i = 0; i < len; ++i) { + terms[i] = RandomPicks.randomFrom(random(), sampleTerms); + if (weird) { + positionsIncrements[i] = random().nextInt(1 << 18); + startOffsets[i] = random().nextInt(); + endOffsets[i] = random().nextInt(); + } else if (i == 0) { + positionsIncrements[i] = _TestUtil.nextInt(random(), 1, 1 << 5); + startOffsets[i] = _TestUtil.nextInt(random(), 0, 1 << 16); + endOffsets[i] = startOffsets[i] + _TestUtil.nextInt(random(), 0, rarely() ? 1 << 10 : 20); + } else { + positionsIncrements[i] = _TestUtil.nextInt(random(), 0, 1 << 5); + startOffsets[i] = startOffsets[i-1] + _TestUtil.nextInt(random(), 0, 1 << 16); + endOffsets[i] = startOffsets[i] + _TestUtil.nextInt(random(), 0, rarely() ? 1 << 10 : 20); + } + } + for (int i = 0; i < len; ++i) { + if (i == 0) { + positions[i] = positionsIncrements[i] - 1; + } else { + positions[i] = positions[i - 1] + positionsIncrements[i]; + } + } + if (rarely()) { + Arrays.fill(payloads, randomPayload()); + } else { + for (int i = 0; i < len; ++i) { + payloads[i] = randomPayload(); + } + } + + positionToTerms = new HashMap>(); + startOffsetToTerms = new HashMap>(); + for (int i = 0; i < len; ++i) { + if (!positionToTerms.containsKey(positions[i])) { + positionToTerms.put(positions[i], new HashSet(1)); + } + positionToTerms.get(positions[i]).add(i); + if (!startOffsetToTerms.containsKey(startOffsets[i])) { + startOffsetToTerms.put(startOffsets[i], new HashSet(1)); + } + startOffsetToTerms.get(startOffsets[i]).add(i); + } + + addAttributeImpl(new PermissiveOffsetAttributeImpl()); + + termAtt = addAttribute(CharTermAttribute.class); + piAtt = addAttribute(PositionIncrementAttribute.class); + oAtt = addAttribute(OffsetAttribute.class); + pAtt = addAttribute(PayloadAttribute.class); + } + + @Override + public final boolean incrementToken() throws IOException { + if (i < terms.length) { + termAtt.setLength(0).append(terms[i]); + piAtt.setPositionIncrement(positionsIncrements[i]); + oAtt.setOffset(startOffsets[i], endOffsets[i]); + pAtt.setPayload(payloads[i]); + ++i; + return true; + } else { + return false; + } + } + + } + + static FieldType randomFieldType() { + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.setStoreTermVectors(true); + ft.setStoreTermVectorPositions(random().nextBoolean()); + ft.setStoreTermVectorOffsets(random().nextBoolean()); + if (random().nextBoolean()) { + ft.setStoreTermVectorPositions(true); + ft.setStoreTermVectorPayloads(true); + } + ft.freeze(); + return ft; + } + + public void testRandomVectors() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30)); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf); + String[] sampleTerms = new String[RandomInts.randomIntBetween(random(), 20, 50)]; + for (int i = 0; i < sampleTerms.length; ++i) { + sampleTerms[i] = _TestUtil.randomUnicodeString(random()); + } + FieldType ft = randomFieldType(); + // generate random documents and index them + final String[] fieldNames = new String[_TestUtil.nextInt(random(), 1, 200)]; + for (int i = 0; i < fieldNames.length; ++i) { + String fieldName; + do { + fieldName = _TestUtil.randomSimpleString(random()); + } while ("id".equals(fieldName)); + fieldNames[i] = fieldName; + } + final int numDocs = _TestUtil.nextInt(random(), 10, 100); + @SuppressWarnings("unchecked") + final Map[] fieldValues = new Map[numDocs]; + for (int i = 0; i < numDocs; ++i) { + fieldValues[i] = new HashMap(); + final int numFields = _TestUtil.nextInt(random(), 0, rarely() ? fieldNames.length : 5); + for (int j = 0; j < numFields; ++j) { + final String fieldName = fieldNames[(i+j*31) % fieldNames.length]; + final int tokenStreamLen = _TestUtil.nextInt(random(), 1, rarely() ? 300 : 5); + fieldValues[i].put(fieldName, new RandomTokenStream(tokenStreamLen, sampleTerms, rarely())); + } + } + + // index them + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + doc.add(new IntField("id", i, Store.YES)); + for (Map.Entry entry : fieldValues[i].entrySet()) { + doc.add(new Field(entry.getKey(), entry.getValue(), ft)); + } + iw.addDocument(doc); + } + + iw.commit(); + // make sure the format can merge + iw.forceMerge(2); + + // read term vectors + final DirectoryReader reader = DirectoryReader.open(dir); + for (int i = 0; i < 100; ++i) { + final int docID = random().nextInt(numDocs); + final Map fvs = fieldValues[reader.document(docID).getField("id").numericValue().intValue()]; + final Fields fields = reader.getTermVectors(docID); + if (fvs.isEmpty()) { + assertNull(fields); + } else { + Set fns = new HashSet(); + for (String field : fields) { + fns.add(field); + } + assertEquals(fields.size(), fns.size()); + assertEquals(fvs.keySet(), fns); + for (String field : fields) { + final RandomTokenStream tk = fvs.get(field); + assert tk != null; + final Terms terms = fields.terms(field); + assertEquals(ft.storeTermVectorPositions(), terms.hasPositions()); + assertEquals(ft.storeTermVectorOffsets(), terms.hasOffsets()); + assertEquals(1, terms.getDocCount()); + final TermsEnum termsEnum = terms.iterator(null); + while (termsEnum.next() != null) { + assertEquals(1, termsEnum.docFreq()); + final DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.docsAndPositions(null, null); + final DocsEnum docsEnum = docsAndPositionsEnum == null ? termsEnum.docs(null, null) : docsAndPositionsEnum; + if (ft.storeTermVectorOffsets() || ft.storeTermVectorPositions()) { + assertNotNull(docsAndPositionsEnum); + } + assertEquals(0, docsEnum.nextDoc()); + if (terms.hasPositions() || terms.hasOffsets()) { + final int freq = docsEnum.freq(); + assertTrue(freq >= 1); + if (docsAndPositionsEnum != null) { + for (int k = 0; k < freq; ++k) { + final int position = docsAndPositionsEnum.nextPosition(); + final Set indexes; + if (terms.hasPositions()) { + indexes = tk.positionToTerms.get(position); + assertNotNull(tk.positionToTerms.keySet().toString() + " does not contain " + position, indexes); + } else { + indexes = tk.startOffsetToTerms.get(docsAndPositionsEnum.startOffset()); + assertNotNull(indexes); + } + if (terms.hasPositions()) { + boolean foundPosition = false; + for (int index : indexes) { + if (new BytesRef(tk.terms[index]).equals(termsEnum.term()) && tk.positions[index] == position) { + foundPosition = true; + break; + } + } + assertTrue(foundPosition); + } + if (terms.hasOffsets()) { + boolean foundOffset = false; + for (int index : indexes) { + if (new BytesRef(tk.terms[index]).equals(termsEnum.term()) && tk.startOffsets[index] == docsAndPositionsEnum.startOffset() && tk.endOffsets[index] == docsAndPositionsEnum.endOffset()) { + foundOffset = true; + break; + } + } + assertTrue(foundOffset); + } + if (terms.hasPayloads()) { + boolean foundPayload = false; + for (int index : indexes) { + if (new BytesRef(tk.terms[index]).equals(termsEnum.term()) && equals(tk.payloads[index], docsAndPositionsEnum.getPayload())) { + foundPayload = true; + break; + } + } + assertTrue(foundPayload); + } + } + } + } + assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc()); + } + } + } + } + IOUtils.close(reader, iw, dir); + } + + private static boolean equals(Object o1, Object o2) { + if (o1 == null) { + return o2 == null; + } else { + return o1.equals(o2); + } + } } Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (révision 1435301) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (copie de travail) @@ -1482,7 +1482,15 @@ doc.add(newField("c", "val", customType)); writer.addDocument(doc); // Adding just one document does not call flush yet. - assertEquals("only the stored and term vector files should exist in the directory", 5 + extraFileCount, dir.listAll().length); + int computedExtraFileCount = 0; + for (String file : dir.listAll()) { + if (file.lastIndexOf('.') < 0 + // don't count stored fields and term vectors in + || !Arrays.asList("fdx", "fdt", "tvx", "tvd", "tvf").contains(file.substring(file.lastIndexOf('.') + 1))) { + ++computedExtraFileCount; + } + } + assertEquals("only the stored and term vector files should exist in the directory", extraFileCount, computedExtraFileCount); doc = new Document(); doc.add(newField("c", "val", customType)); Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (copie de travail) @@ -0,0 +1,1048 @@ +package org.apache.lucene.codecs.compressing; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.BLOCK_SIZE; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_DAT; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_IDX; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.FLAGS_BITS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.OFFSETS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PAYLOADS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.POSITIONS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_EXTENSION; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.TermVectorsReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.packed.BlockPackedReader; +import org.apache.lucene.util.packed.PackedInts; + + +/** + * {@link TermVectorsReader} for {@link CompressingTermVectorsFormat}. + * @lucene.experimental + */ +public final class CompressingTermVectorsReader extends TermVectorsReader implements Closeable { + + private final FieldInfos fieldInfos; + final CompressingStoredFieldsIndexReader indexReader; + final IndexInput vectorsStream; + private final int packedIntsVersion; + private final CompressionMode compressionMode; + private final Decompressor decompressor; + private final int chunkSize; + private final int numDocs; + private boolean closed; + private final BlockPackedReader reader; + + // used by clone + private CompressingTermVectorsReader(CompressingTermVectorsReader reader) { + this.fieldInfos = reader.fieldInfos; + this.vectorsStream = reader.vectorsStream.clone(); + this.indexReader = reader.indexReader.clone(); + this.packedIntsVersion = reader.packedIntsVersion; + this.compressionMode = reader.compressionMode; + this.decompressor = reader.decompressor.clone(); + this.chunkSize = reader.chunkSize; + this.numDocs = reader.numDocs; + this.reader = new BlockPackedReader(vectorsStream, packedIntsVersion, BLOCK_SIZE, 0); + this.closed = false; + } + + /** Sole constructor. */ + public CompressingTermVectorsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn, + IOContext context, String formatName, CompressionMode compressionMode) throws IOException { + this.compressionMode = compressionMode; + final String segment = si.name; + boolean success = false; + fieldInfos = fn; + numDocs = si.getDocCount(); + IndexInput indexStream = null; + try { + vectorsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION); + indexStream = d.openInput(indexStreamFN, context); + + final String codecNameIdx = formatName + CODEC_SFX_IDX; + final String codecNameDat = formatName + CODEC_SFX_DAT; + CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT); + CodecUtil.checkHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT); + assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer(); + assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); + + indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); + indexStream = null; + + packedIntsVersion = vectorsStream.readVInt(); + chunkSize = vectorsStream.readVInt(); + decompressor = compressionMode.newDecompressor(); + this.reader = new BlockPackedReader(vectorsStream, packedIntsVersion, BLOCK_SIZE, 0); + + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(this, indexStream); + } + } + } + + CompressionMode getCompressionMode() { + return compressionMode; + } + + int getChunkSize() { + return chunkSize; + } + + int getPackedIntsVersion() { + return packedIntsVersion; + } + + CompressingStoredFieldsIndexReader getIndex() { + return indexReader; + } + + IndexInput getVectorsStream() { + return vectorsStream; + } + + /** + * @throws AlreadyClosedException if this TermVectorsReader is closed + */ + private void ensureOpen() throws AlreadyClosedException { + if (closed) { + throw new AlreadyClosedException("this FieldsReader is closed"); + } + } + + @Override + public void close() throws IOException { + if (!closed) { + IOUtils.close(vectorsStream, indexReader); + closed = true; + } + } + + @Override + public TermVectorsReader clone() { + return new CompressingTermVectorsReader(this); + } + + @Override + public Fields get(int doc) throws IOException { + ensureOpen(); + + // seek to the right place + { + final long startPointer = indexReader.getStartPointer(doc); + vectorsStream.seek(startPointer); + } + + // decode + // - docBase: first doc ID of the chunk + // - chunkDocs: number of docs of the chunk + final int docBase = vectorsStream.readVInt(); + final int chunkDocs = vectorsStream.readVInt(); + if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs) { + throw new CorruptIndexException("docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc); + } + + final int skip; // number of fields to skip + final int numFields; // number of fields of the document we're looking for + final int totalFields; // total number of fields of the chunk (sum for all docs) + if (chunkDocs == 1) { + skip = 0; + numFields = totalFields = vectorsStream.readVInt(); + } else { + reader.reset(vectorsStream, chunkDocs); + int sum = 0; + for (int i = docBase; i < doc; ++i) { + sum += reader.next(); + } + skip = sum; + numFields = (int) reader.next(); + sum += numFields; + for (int i = doc + 1; i < docBase + chunkDocs; ++i) { + sum += reader.next(); + } + totalFields = sum; + } + + if (numFields == 0) { + // no vectors + return null; + } + + // read field numbers that have term vectors + final int[] fieldNums; + { + final int token = vectorsStream.readByte() & 0xFF; + assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0 + final int bitsPerFieldNum = token & 0x1F; + int totalDistinctFields = token >>> 5; + if (totalDistinctFields == 0x07) { + totalDistinctFields += vectorsStream.readVInt(); + } + ++totalDistinctFields; + final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalDistinctFields, bitsPerFieldNum, 1); + fieldNums = new int[totalDistinctFields]; + for (int i = 0; i < totalDistinctFields; ++i) { + fieldNums[i] = (int) it.next(); + } + } + + // read field numbers and flags + final int[] fieldNumOffs = new int[numFields]; + final PackedInts.Reader flags; + { + final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1); + final PackedInts.Reader allFieldNumOffs = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff); + switch (vectorsStream.readVInt()) { + case 0: + final PackedInts.Reader fieldFlags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, fieldNums.length, FLAGS_BITS); + PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT); + for (int i = 0; i < totalFields; ++i) { + final int fieldNumOff = (int) allFieldNumOffs.get(i); + assert fieldNumOff >= 0 && fieldNumOff < fieldNums.length; + final int fgs = (int) fieldFlags.get(fieldNumOff); + f.set(i, fgs); + } + flags = f; + break; + case 1: + flags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, FLAGS_BITS); + break; + default: + throw new AssertionError(); + } + for (int i = 0; i < numFields; ++i) { + fieldNumOffs[i] = (int) allFieldNumOffs.get(skip + i); + } + } + + // number of terms per field for all fields + final PackedInts.Reader numTerms; + final int totalTerms; + { + final int bitsRequired = vectorsStream.readVInt(); + numTerms = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsRequired); + int sum = 0; + for (int i = 0; i < totalFields; ++i) { + sum += numTerms.get(i); + } + totalTerms = sum; + } + + // term lengths + int docOff = 0, docLen = 0, totalLen; + final int[] fieldLengths = new int[numFields]; + final int[][] prefixLengths = new int[numFields][]; + final int[][] suffixLengths = new int[numFields][]; + { + reader.reset(vectorsStream, totalTerms); + // skip + int toSkip = 0; + for (int i = 0; i < skip; ++i) { + toSkip += numTerms.get(i); + } + reader.skip(toSkip); + // read prefix lengths + for (int i = 0; i < numFields; ++i) { + final int termCount = (int) numTerms.get(skip + i); + final int[] fieldPrefixLengths = new int[termCount]; + prefixLengths[i] = fieldPrefixLengths; + for (int j = 0; j < termCount; ) { + final LongsRef next = reader.next(termCount - j); + for (int k = 0; k < next.length; ++k) { + fieldPrefixLengths[j++] = (int) next.longs[next.offset + k]; + } + } + } + reader.skip(totalTerms - reader.ord()); + + reader.reset(vectorsStream, totalTerms); + // skip + toSkip = 0; + for (int i = 0; i < skip; ++i) { + for (int j = 0; j < numTerms.get(i); ++j) { + docOff += reader.next(); + } + } + for (int i = 0; i < numFields; ++i) { + final int termCount = (int) numTerms.get(skip + i); + final int[] fieldSuffixLengths = new int[termCount]; + suffixLengths[i] = fieldSuffixLengths; + for (int j = 0; j < termCount; ) { + final LongsRef next = reader.next(termCount - j); + for (int k = 0; k < next.length; ++k) { + fieldSuffixLengths[j++] = (int) next.longs[next.offset + k]; + } + } + fieldLengths[i] = sum(suffixLengths[i]); + docLen += fieldLengths[i]; + } + totalLen = docOff + docLen; + for (int i = skip + numFields; i < totalFields; ++i) { + for (int j = 0; j < numTerms.get(i); ++j) { + totalLen += reader.next(); + } + } + } + + // term freqs + final int[] termFreqs = new int[totalTerms]; + { + reader.reset(vectorsStream, totalTerms); + for (int i = 0; i < totalTerms; ) { + final LongsRef next = reader.next(totalTerms - i); + for (int k = 0; k < next.length; ++k) { + termFreqs[i++] = 1 + (int) next.longs[next.offset + k]; + } + } + } + + // total number of positions, offsets and payloads + int totalPositions = 0, totalOffsets = 0, totalPayloads = 0; + for (int i = 0, termIndex = 0; i < totalFields; ++i) { + final int f = (int) flags.get(i); + final int termCount = (int) numTerms.get(i); + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex++]; + if ((f & POSITIONS) != 0) { + totalPositions += freq; + } + if ((f & OFFSETS) != 0) { + totalOffsets += freq; + } + if ((f & PAYLOADS) != 0) { + totalPayloads += freq; + } + } + assert i != totalFields - 1 || termIndex == totalTerms : termIndex + " " + totalTerms; + } + + final int[][] positionIndex = positionIndex(skip, numFields, numTerms, termFreqs); + final int[][] positions, startOffsets, lengths; + if (totalPositions > 0) { + positions = readPositions(skip, numFields, flags, numTerms, termFreqs, POSITIONS, totalPositions, positionIndex); + } else { + positions = new int[numFields][]; + } + + if (totalOffsets > 0) { + // average number of chars per term + final float[] charsPerTerm = new float[fieldNums.length]; + for (int i = 0; i < charsPerTerm.length; ++i) { + charsPerTerm[i] = Float.intBitsToFloat(vectorsStream.readInt()); + } + startOffsets = readPositions(skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex); + lengths = readPositions(skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex); + + for (int i = 0; i < numFields; ++i) { + final int[] fStartOffsets = startOffsets[i]; + final int[] fPositions = positions[i]; + // patch offsets from positions + if (fStartOffsets != null && fPositions != null) { + final float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]]; + for (int j = 0; j < startOffsets[i].length; ++j) { + fStartOffsets[j] += (int) (fieldCharsPerTerm * fPositions[j]); + } + } + if (fStartOffsets != null) { + final int[] fPrefixLengths = prefixLengths[i]; + final int[] fSuffixLengths = suffixLengths[i]; + final int[] fLengths = lengths[i]; + for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) { + // delta-decode start offsets and patch lengths using term lengths + final int termLength = fPrefixLengths[j] + fSuffixLengths[j]; + lengths[i][positionIndex[i][j]] += termLength; + for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k) { + fStartOffsets[k] += fStartOffsets[k - 1]; + fLengths[k] += termLength; + } + } + } + } + } else { + startOffsets = lengths = new int[numFields][]; + } + if (totalPositions > 0) { + // delta-decode positions + for (int i = 0; i < numFields; ++i) { + final int[] fPositions = positions[i]; + final int[] fpositionIndex = positionIndex[i]; + if (fPositions != null) { + for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) { + // delta-decode start offsets + for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k) { + fPositions[k] += fPositions[k - 1]; + } + } + } + } + } + + // payload lengths + final int[][] payloadIndex = new int[numFields][]; + int totalPayloadLength = 0; + int payloadOff = 0; + int payloadLen = 0; + if (totalPayloads > 0) { + reader.reset(vectorsStream, totalPayloads); + // skip + int termIndex = 0; + for (int i = 0; i < skip; ++i) { + final int f = (int) flags.get(i); + final int termCount = (int) numTerms.get(i); + if ((f & PAYLOADS) != 0) { + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex + j]; + for (int k = 0; k < freq; ++k) { + final int l = (int) reader.next(); + payloadOff += l; + } + } + } + termIndex += termCount; + } + totalPayloadLength = payloadOff; + // read doc payload lengths + for (int i = 0; i < numFields; ++i) { + final int f = (int) flags.get(skip + i); + final int termCount = (int) numTerms.get(skip + i); + if ((f & PAYLOADS) != 0) { + final int totalFreq = positionIndex[i][termCount]; + payloadIndex[i] = new int[totalFreq + 1]; + int posIdx = 0; + payloadIndex[i][posIdx] = payloadLen; + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex + j]; + for (int k = 0; k < freq; ++k) { + final int payloadLength = (int) reader.next(); + payloadLen += payloadLength; + payloadIndex[i][posIdx+1] = payloadLen; + ++posIdx; + } + } + assert posIdx == totalFreq; + } + termIndex += termCount; + } + totalPayloadLength += payloadLen; + for (int i = skip + numFields; i < totalFields; ++i) { + final int f = (int) flags.get(i); + final int termCount = (int) numTerms.get(i); + if ((f & PAYLOADS) != 0) { + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex + j]; + for (int k = 0; k < freq; ++k) { + totalPayloadLength += reader.next(); + } + } + } + termIndex += termCount; + } + assert termIndex == totalTerms : termIndex + " " + totalTerms; + } + + // decompress data + final BytesRef suffixBytes = new BytesRef(); + decompressor.decompress(vectorsStream, totalLen + totalPayloadLength, docOff + payloadOff, docLen + payloadLen, suffixBytes); + suffixBytes.length = docLen; + final BytesRef payloadBytes = new BytesRef(suffixBytes.bytes, suffixBytes.offset + docLen, payloadLen); + + final int[] fieldFlags = new int[numFields]; + for (int i = 0; i < numFields; ++i) { + fieldFlags[i] = (int) flags.get(skip + i); + } + + final int[] fieldNumTerms = new int[numFields]; + for (int i = 0; i < numFields; ++i) { + fieldNumTerms[i] = (int) numTerms.get(skip + i); + } + + final int[][] fieldTermFreqs = new int[numFields][]; + { + int termIdx = 0; + for (int i = 0; i < skip; ++i) { + termIdx += numTerms.get(i); + } + for (int i = 0; i < numFields; ++i) { + final int termCount = (int) numTerms.get(skip + i); + fieldTermFreqs[i] = new int[termCount]; + for (int j = 0; j < termCount; ++j) { + fieldTermFreqs[i][j] = termFreqs[termIdx++]; + } + } + } + + assert sum(fieldLengths) == docLen : sum(fieldLengths) + " != " + docLen; + + return new TVFields(fieldNums, fieldFlags, fieldNumOffs, fieldNumTerms, fieldLengths, + prefixLengths, suffixLengths, fieldTermFreqs, + positionIndex, positions, startOffsets, lengths, + payloadBytes, payloadIndex, + suffixBytes); + } + + // field -> term index -> position index + private int[][] positionIndex(int skip, int numFields, PackedInts.Reader numTerms, int[] termFreqs) { + final int[][] positionIndex = new int[numFields][]; + int termIndex = 0; + for (int i = 0; i < skip; ++i) { + final int termCount = (int) numTerms.get(i); + termIndex += termCount; + } + for (int i = 0; i < numFields; ++i) { + final int termCount = (int) numTerms.get(skip + i); + positionIndex[i] = new int[termCount + 1]; + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex+j]; + positionIndex[i][j + 1] = positionIndex[i][j] + freq; + } + termIndex += termCount; + } + return positionIndex; + } + + private int[][] readPositions(int skip, int numFields, PackedInts.Reader flags, PackedInts.Reader numTerms, int[] termFreqs, int flag, final int totalPositions, int[][] positionIndex) throws IOException { + final int[][] positions = new int[numFields][]; + reader.reset(vectorsStream, totalPositions); + // skip + int toSkip = 0; + int termIndex = 0; + for (int i = 0; i < skip; ++i) { + final int f = (int) flags.get(i); + final int termCount = (int) numTerms.get(i); + if ((f & flag) != 0) { + for (int j = 0; j < termCount; ++j) { + final int freq = termFreqs[termIndex+j]; + toSkip += freq; + } + } + termIndex += termCount; + } + reader.skip(toSkip); + // read doc positions + for (int i = 0; i < numFields; ++i) { + final int f = (int) flags.get(skip + i); + final int termCount = (int) numTerms.get(skip + i); + if ((f & flag) != 0) { + final int totalFreq = positionIndex[i][termCount]; + final int[] fieldPositions = new int[totalFreq]; + positions[i] = fieldPositions; + for (int j = 0; j < totalFreq; ) { + final LongsRef nextPositions = reader.next(totalFreq - j); + for (int k = 0; k < nextPositions.length; ++k) { + fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k]; + } + } + } + termIndex += termCount; + } + reader.skip(totalPositions - reader.ord()); + return positions; + } + + private class TVFields extends Fields { + + private final int[] fieldNums, fieldFlags, fieldNumOffs, numTerms, fieldLengths; + private final int[][] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex; + private final BytesRef suffixBytes, payloadBytes; + + public TVFields(int[] fieldNums, int[] fieldFlags, int[] fieldNumOffs, int[] numTerms, int[] fieldLengths, + int[][] prefixLengths, int[][] suffixLengths, int[][] termFreqs, + int[][] positionIndex, int[][] positions, int[][] startOffsets, int[][] lengths, + BytesRef payloadBytes, int[][] payloadIndex, + BytesRef suffixBytes) { + this.fieldNums = fieldNums; + this.fieldFlags = fieldFlags; + this.fieldNumOffs = fieldNumOffs; + this.numTerms = numTerms; + this.fieldLengths = fieldLengths; + this.prefixLengths = prefixLengths; + this.suffixLengths = suffixLengths; + this.termFreqs = termFreqs; + this.positionIndex = positionIndex; + this.positions = positions; + this.startOffsets = startOffsets; + this.lengths = lengths; + this.payloadBytes = payloadBytes; + this.payloadIndex = payloadIndex; + this.suffixBytes = suffixBytes; + } + + @Override + public Iterator iterator() { + return new Iterator() { + int i = 0; + @Override + public boolean hasNext() { + return i < fieldNumOffs.length; + } + @Override + public String next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final int fieldNum = fieldNums[fieldNumOffs[i++]]; + return fieldInfos.fieldInfo(fieldNum).name; + } + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public Terms terms(String field) throws IOException { + final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + if (fieldInfo == null) { + return null; + } + int idx = -1; + for (int i = 0; i < fieldNumOffs.length; ++i) { + if (fieldNums[fieldNumOffs[i]] == fieldInfo.number) { + idx = i; + break; + } + } + + if (idx == -1 || numTerms[idx] == 0) { + // no term + return null; + } + int fieldOff = 0, fieldLen = -1; + for (int i = 0; i < fieldNumOffs.length; ++i) { + if (i < idx) { + fieldOff += fieldLengths[i]; + } else { + fieldLen = fieldLengths[i]; + break; + } + } + assert fieldLen >= 0; + return new TVTerms(numTerms[idx], fieldFlags[idx], + prefixLengths[idx], suffixLengths[idx], termFreqs[idx], + positionIndex[idx], positions[idx], startOffsets[idx], lengths[idx], + payloadIndex[idx], payloadBytes, + new BytesRef(suffixBytes.bytes, suffixBytes.offset + fieldOff, fieldLen)); + } + + @Override + public int size() { + return fieldNumOffs.length; + } + + } + + private class TVTerms extends Terms { + + private final int numTerms, flags; + private final int[] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex; + private final BytesRef termBytes, payloadBytes; + + TVTerms(int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, + int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths, + int[] payloadIndex, BytesRef payloadBytes, + BytesRef termBytes) { + this.numTerms = numTerms; + this.flags = flags; + this.prefixLengths = prefixLengths; + this.suffixLengths = suffixLengths; + this.termFreqs = termFreqs; + this.positionIndex = positionIndex; + this.positions = positions; + this.startOffsets = startOffsets; + this.lengths = lengths; + this.payloadIndex = payloadIndex; + this.payloadBytes = payloadBytes; + this.termBytes = termBytes; + } + + @Override + public TermsEnum iterator(TermsEnum reuse) throws IOException { + final TVTermsEnum termsEnum; + if (reuse != null && reuse instanceof TVTermsEnum) { + termsEnum = (TVTermsEnum) reuse; + } else { + termsEnum = new TVTermsEnum(); + } + termsEnum.reset(numTerms, flags, prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, + payloadIndex, payloadBytes, + new ByteArrayDataInput(termBytes.bytes, termBytes.offset, termBytes.length)); + return termsEnum; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public long size() throws IOException { + return numTerms; + } + + @Override + public long getSumTotalTermFreq() throws IOException { + return -1L; + } + + @Override + public long getSumDocFreq() throws IOException { + return numTerms; + } + + @Override + public int getDocCount() throws IOException { + return 1; + } + + @Override + public boolean hasOffsets() { + return (flags & OFFSETS) != 0; + } + + @Override + public boolean hasPositions() { + return (flags & POSITIONS) != 0; + } + + @Override + public boolean hasPayloads() { + return (flags & PAYLOADS) != 0; + } + + } + + private static class TVTermsEnum extends TermsEnum { + + private int numTerms, startPos, ord; + private int[] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex; + private ByteArrayDataInput in; + private BytesRef payloads; + private final BytesRef term; + + private TVTermsEnum() { + term = new BytesRef(16); + } + + void reset(int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths, + int[] payloadIndex, BytesRef payloads, ByteArrayDataInput in) { + this.numTerms = numTerms; + this.prefixLengths = prefixLengths; + this.suffixLengths = suffixLengths; + this.termFreqs = termFreqs; + this.positionIndex = positionIndex; + this.positions = positions; + this.startOffsets = startOffsets; + this.lengths = lengths; + this.payloadIndex = payloadIndex; + this.payloads = payloads; + this.in = in; + startPos = in.getPosition(); + reset(); + } + + void reset() { + term.length = 0; + in.setPosition(startPos); + ord = -1; + } + + @Override + public BytesRef next() throws IOException { + if (ord == numTerms - 1) { + return null; + } else { + assert ord < numTerms; + ++ord; + } + + // read term + term.offset = 0; + term.length = prefixLengths[ord] + suffixLengths[ord]; + if (term.length > term.bytes.length) { + term.bytes = ArrayUtil.grow(term.bytes, term.length); + } + in.readBytes(term.bytes, prefixLengths[ord], suffixLengths[ord]); + + return term; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public SeekStatus seekCeil(BytesRef text, boolean useCache) + throws IOException { + if (ord < numTerms && ord >= 0) { + final int cmp = term().compareTo(text); + if (cmp == 0) { + return SeekStatus.FOUND; + } else if (cmp > 0) { + reset(); + } + } + // linear scan + do { + next(); + } while (ord < numTerms - 1 && term().compareTo(text) < 0); + return term().equals(text) ? SeekStatus.FOUND : SeekStatus.END; + } + + @Override + public void seekExact(long ord) throws IOException { + if (ord < -1 || ord >= numTerms) { + throw new IOException("ord is out of range: ord=" + ord + ", numTerms=" + numTerms); + } + if (ord < this.ord) { + reset(); + } + for (int i = this.ord; i < ord; ++i) { + next(); + } + assert ord == this.ord(); + } + + @Override + public BytesRef term() throws IOException { + return term; + } + + @Override + public long ord() throws IOException { + return ord; + } + + @Override + public int docFreq() throws IOException { + return 1; + } + + @Override + public long totalTermFreq() throws IOException { + return termFreqs[ord]; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) + throws IOException { + return docsAndPositions(liveDocs, reuse, flags); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, + DocsAndPositionsEnum reuse, int flags) throws IOException { + if ((flags & POSITIONS) == 0 && (flags & OFFSETS) == 0) { + return null; + } + return docsAndPositions(liveDocs, (DocsEnum) reuse, flags); + } + + private DocsAndPositionsEnum docsAndPositions(Bits liveDocs, + DocsEnum reuse, int ignoredFlags) throws IOException { + final TVDocsEnum docsEnum; + if (reuse != null && reuse instanceof TVDocsEnum) { + docsEnum = (TVDocsEnum) reuse; + } else { + docsEnum = new TVDocsEnum(); + } + + docsEnum.reset(liveDocs, termFreqs[ord], positionIndex[ord], positions, startOffsets, lengths, payloads, payloadIndex); + return docsEnum; + } + + } + + private static class TVDocsEnum extends DocsAndPositionsEnum { + + private Bits liveDocs; + private int doc = -1; + private int termFreq; + private int positionIndex; + private int[] positions; + private int[] startOffsets; + private int[] lengths; + private final BytesRef payload; + private int[] payloadIndex; + private int basePayloadOffset; + private int i; + + TVDocsEnum() { + payload = new BytesRef(); + } + + public void reset(Bits liveDocs, int freq, int positionIndex, int[] positions, + int[] startOffsets, int[] lengths, BytesRef payloads, + int[] payloadIndex) { + this.liveDocs = liveDocs; + this.termFreq = freq; + this.positionIndex = positionIndex; + this.positions = positions; + this.startOffsets = startOffsets; + this.lengths = lengths; + this.basePayloadOffset = payloads.offset; + this.payload.bytes = payloads.bytes; + payload.offset = payload.length = 0; + this.payloadIndex = payloadIndex; + + doc = i = -1; + } + + private void checkDoc() { + if (doc == NO_MORE_DOCS) { + throw new IllegalStateException("DocsEnum exhausted"); + } else if (doc == -1) { + throw new IllegalStateException("DocsEnum not started"); + } + } + + private void checkPosition() { + checkDoc(); + if (i < 0) { + throw new IllegalStateException("Position enum not started"); + } else if (i >= termFreq) { + throw new IllegalStateException("Read past last position"); + } + } + + @Override + public int nextPosition() throws IOException { + if (doc != 0) { + throw new IllegalStateException(); + } else if (i >= termFreq - 1) { + throw new IllegalStateException("Read past last position"); + } + + ++i; + + if (payloadIndex != null) { + payload.offset = basePayloadOffset + payloadIndex[positionIndex + i]; + payload.length = payloadIndex[positionIndex + i + 1] - payloadIndex[positionIndex + i]; + } + + if (positions == null) { + return -1; + } else { + return positions[positionIndex + i]; + } + } + + @Override + public int startOffset() throws IOException { + checkPosition(); + if (startOffsets == null) { + return -1; + } else { + return startOffsets[positionIndex + i]; + } + } + + @Override + public int endOffset() throws IOException { + checkPosition(); + if (startOffsets == null) { + return -1; + } else { + return startOffsets[positionIndex + i] + lengths[positionIndex + i]; + } + } + + @Override + public BytesRef getPayload() throws IOException { + checkPosition(); + if (payloadIndex == null || payload.length == 0) { + return null; + } else { + return payload; + } + } + + @Override + public int freq() throws IOException { + checkDoc(); + return termFreq; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int nextDoc() throws IOException { + if (doc == -1 && (liveDocs == null || liveDocs.get(0))) { + return (doc = 0); + } else { + return (doc = NO_MORE_DOCS); + } + } + + @Override + public int advance(int target) throws IOException { + if (doc == -1 && target == 0 && (liveDocs == null || liveDocs.get(0))) { + return (doc = 0); + } else { + return (doc = NO_MORE_DOCS); + } + } + + } + + private static int sum(int[] arr) { + int sum = 0; + for (int el : arr) { + sum += el; + } + return sum; + } + +} Modification de propriétés sur lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (copie de travail) @@ -0,0 +1,818 @@ +package org.apache.lucene.codecs.compressing; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Deque; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.TermVectorsReader; +import org.apache.lucene.codecs.TermVectorsWriter; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.packed.BlockPackedWriter; +import org.apache.lucene.util.packed.PackedInts; + +/** + * {@link TermVectorsWriter} for {@link CompressingTermVectorsFormat}. + * @lucene.experimental + */ +public final class CompressingTermVectorsWriter extends TermVectorsWriter { + + static final String VECTORS_EXTENSION = "tvd"; + static final String VECTORS_INDEX_EXTENSION = "tvx"; + + static final String CODEC_SFX_IDX = "Index"; + static final String CODEC_SFX_DAT = "Data"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + static final int BLOCK_SIZE = 64; + + static final int POSITIONS = 0x01; + static final int OFFSETS = 0x02; + static final int PAYLOADS = 0x04; + static final int FLAGS_BITS = PackedInts.bitsRequired(POSITIONS | OFFSETS | PAYLOADS); + + private final Directory directory; + private final String segment; + private final String segmentSuffix; + private CompressingStoredFieldsIndexWriter indexWriter; + private IndexOutput vectorsStream; + + private final CompressionMode compressionMode; + private final Compressor compressor; + private final int chunkSize; + + /** a pending doc */ + private class DocData { + final int numFields; + final Deque fields; + final int posStart, offStart, payStart; + DocData(int numFields, int posStart, int offStart, int payStart) { + this.numFields = numFields; + this.fields = new ArrayDeque(numFields); + this.posStart = posStart; + this.offStart = offStart; + this.payStart = payStart; + } + FieldData addField(int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads) { + final FieldData field; + if (fields.isEmpty()) { + field = new FieldData(fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart); + } else { + final FieldData last = fields.getLast(); + final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0); + final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0); + final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0); + field = new FieldData(fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart); + } + fields.add(field); + return field; + } + } + + private DocData addDocData(int numVectorFields) { + FieldData last = null; + for (Iterator it = pendingDocs.descendingIterator(); it.hasNext(); ) { + final DocData doc = it.next(); + if (!doc.fields.isEmpty()) { + last = doc.fields.getLast(); + break; + } + } + final DocData doc; + if (last == null) { + doc = new DocData(numVectorFields, 0, 0, 0); + } else { + final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0); + final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0); + final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0); + doc = new DocData(numVectorFields, posStart, offStart, payStart); + } + pendingDocs.add(doc); + return doc; + } + + /** a pending field */ + private class FieldData { + final boolean hasPositions, hasOffsets, hasPayloads; + final int fieldNum, flags, numTerms; + final int[] freqs, prefixLengths, suffixLengths; + final int posStart, offStart, payStart; + int totalPositions; + int ord; + FieldData(int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads, + int posStart, int offStart, int payStart) { + this.fieldNum = fieldNum; + this.numTerms = numTerms; + this.hasPositions = positions; + this.hasOffsets = offsets; + this.hasPayloads = payloads; + this.flags = (positions ? POSITIONS : 0) | (offsets ? OFFSETS : 0) | (payloads ? PAYLOADS : 0); + this.freqs = new int[numTerms]; + this.prefixLengths = new int[numTerms]; + this.suffixLengths = new int[numTerms]; + this.posStart = posStart; + this.offStart = offStart; + this.payStart = payStart; + totalPositions = 0; + ord = 0; + } + void addTerm(int freq, int prefixLength, int suffixLength) { + freqs[ord] = freq; + prefixLengths[ord] = prefixLength; + suffixLengths[ord] = suffixLength; + ++ord; + } + void addPosition(int position, int startOffset, int length, int payloadLength) { + if (hasPositions) { + if (posStart + totalPositions == positionsBuf.length) { + positionsBuf = ArrayUtil.grow(positionsBuf); + } + positionsBuf[posStart + totalPositions] = position; + } + if (hasOffsets) { + if (offStart + totalPositions == startOffsetsBuf.length) { + final int newLength = ArrayUtil.oversize(offStart + totalPositions, 4); + startOffsetsBuf = Arrays.copyOf(startOffsetsBuf, newLength); + lengthsBuf = Arrays.copyOf(lengthsBuf, newLength); + } + startOffsetsBuf[offStart + totalPositions] = startOffset; + lengthsBuf[offStart + totalPositions] = length; + } + if (hasPayloads) { + if (payStart + totalPositions == payloadLengthsBuf.length) { + payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf); + } + payloadLengthsBuf[payStart + totalPositions] = payloadLength; + } + ++totalPositions; + } + } + + private int numDocs; // total number of docs seen + private final Deque pendingDocs; // pending docs + private DocData curDoc; // current document + private FieldData curField; // current field + private final BytesRef lastTerm; + private int[] positionsBuf, startOffsetsBuf, lengthsBuf, payloadLengthsBuf; + private final GrowableByteArrayDataOutput termSuffixes; // buffered term suffixes + private final GrowableByteArrayDataOutput payloadBytes; // buffered term payloads + private final BlockPackedWriter writer; + + /** Sole constructor. */ + public CompressingTermVectorsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context, + String formatName, CompressionMode compressionMode, int chunkSize) throws IOException { + assert directory != null; + this.directory = directory; + this.segment = si.name; + this.segmentSuffix = segmentSuffix; + this.compressionMode = compressionMode; + this.compressor = compressionMode.newCompressor(); + this.chunkSize = chunkSize; + + numDocs = 0; + pendingDocs = new ArrayDeque(); + termSuffixes = new GrowableByteArrayDataOutput(ArrayUtil.oversize(chunkSize, 1)); + payloadBytes = new GrowableByteArrayDataOutput(ArrayUtil.oversize(1, 1)); + lastTerm = new BytesRef(ArrayUtil.oversize(30, 1)); + + boolean success = false; + IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION), context); + try { + vectorsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context); + + final String codecNameIdx = formatName + CODEC_SFX_IDX; + final String codecNameDat = formatName + CODEC_SFX_DAT; + CodecUtil.writeHeader(indexStream, codecNameIdx, VERSION_CURRENT); + CodecUtil.writeHeader(vectorsStream, codecNameDat, VERSION_CURRENT); + assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer(); + assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); + + indexWriter = new CompressingStoredFieldsIndexWriter(indexStream); + indexStream = null; + + vectorsStream.writeVInt(PackedInts.VERSION_CURRENT); + vectorsStream.writeVInt(chunkSize); + writer = new BlockPackedWriter(vectorsStream, BLOCK_SIZE); + + positionsBuf = new int[1024]; + startOffsetsBuf = new int[1024]; + lengthsBuf = new int[1024]; + payloadLengthsBuf = new int[1024]; + + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(indexStream); + abort(); + } + } + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(vectorsStream, indexWriter); + } finally { + vectorsStream = null; + indexWriter = null; + } + } + + @Override + public void abort() { + IOUtils.closeWhileHandlingException(this); + IOUtils.deleteFilesIgnoringExceptions(directory, + IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), + IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION)); + } + + @Override + public void startDocument(int numVectorFields) throws IOException { + curDoc = addDocData(numVectorFields); + } + + @Override + public void finishDocument() throws IOException { + // append the payload bytes of the doc after its terms + termSuffixes.writeBytes(payloadBytes.bytes, payloadBytes.length); + payloadBytes.length = 0; + ++numDocs; + if (triggerFlush()) { + flush(); + } + curDoc = null; + } + + @Override + public void startField(FieldInfo info, int numTerms, boolean positions, + boolean offsets, boolean payloads) throws IOException { + curField = curDoc.addField(info.number, numTerms, positions, offsets, payloads); + lastTerm.length = 0; + } + + @Override + public void finishField() throws IOException { + curField = null; + } + + @Override + public void startTerm(BytesRef term, int freq) throws IOException { + assert freq >= 1; + final int prefix = StringHelper.bytesDifference(lastTerm, term); + curField.addTerm(freq, prefix, term.length - prefix); + termSuffixes.writeBytes(term.bytes, term.offset + prefix, term.length - prefix); + // copy last term + if (lastTerm.bytes.length < term.length) { + lastTerm.bytes = new byte[ArrayUtil.oversize(term.length, 1)]; + } + lastTerm.offset = 0; + lastTerm.length = term.length; + System.arraycopy(term.bytes, term.offset, lastTerm.bytes, 0, term.length); + } + + @Override + public void addPosition(int position, int startOffset, int endOffset, + BytesRef payload) throws IOException { + assert curField.flags != 0; + curField.addPosition(position, startOffset, endOffset - startOffset, payload == null ? 0 : payload.length); + if (curField.hasPayloads && payload != null) { + payloadBytes.writeBytes(payload.bytes, payload.offset, payload.length); + } + } + + private boolean triggerFlush() { + return termSuffixes.length >= chunkSize || pendingDocs.size() >= chunkSize; + } + + private void flush() throws IOException { + final int chunkDocs = pendingDocs.size(); + assert chunkDocs > 0 : chunkDocs; + + // write the index file + indexWriter.writeIndex(chunkDocs, vectorsStream.getFilePointer()); + + final int docBase = numDocs - chunkDocs; + vectorsStream.writeVInt(docBase); + vectorsStream.writeVInt(chunkDocs); + + // total number of fields of the chunk + final int totalFields = flushNumFields(chunkDocs); + + if (totalFields > 0) { + // unique field numbers (sorted) + final int[] fieldNums = flushFieldNums(); + // offsets in the array of unique field numbers + flushFields(totalFields, fieldNums); + // flags (does the field have positions, offsets, payloads?) + flushFlags(totalFields, fieldNums); + // number of terms of each field + flushNumTerms(totalFields); + // prefix and suffix lengths for each field + flushTermLengths(); + // term freqs - 1 (because termFreq is always >=1) for each term + flushTermFreqs(); + // positions for all terms, when enabled + flushPositions(); + // offsets for all terms, when enabled + flushOffsets(fieldNums); + // payload lengths for all terms, when enabled + flushPayloadLengths(); + + // compress terms and payloads and write them to the output + compressor.compress(termSuffixes.bytes, 0, termSuffixes.length, vectorsStream); + } + + // reset + pendingDocs.clear(); + curDoc = null; + curField = null; + termSuffixes.length = 0; + } + + private int flushNumFields(int chunkDocs) throws IOException { + if (chunkDocs == 1) { + final int numFields = pendingDocs.getFirst().numFields; + vectorsStream.writeVInt(numFields); + return numFields; + } else { + writer.reset(vectorsStream); + int totalFields = 0; + for (DocData dd : pendingDocs) { + writer.add(dd.numFields); + totalFields += dd.numFields; + } + writer.finish(); + return totalFields; + } + } + + /** Returns a sorted array containing unique field numbers */ + private int[] flushFieldNums() throws IOException { + SortedSet fieldNums = new TreeSet(); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + fieldNums.add(fd.fieldNum); + } + } + + final int numDistinctFields = fieldNums.size(); + assert numDistinctFields > 0; + final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); + final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; + vectorsStream.writeByte((byte) token); + if (numDistinctFields - 1 >= 0x07) { + vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); + } + final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); + for (Integer fieldNum : fieldNums) { + writer.add(fieldNum); + } + writer.finish(); + + int[] fns = new int[fieldNums.size()]; + int i = 0; + for (Integer key : fieldNums) { + fns[i++] = key; + } + return fns; + } + + private void flushFields(int totalFields, int[] fieldNums) throws IOException { + final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); + assert fieldNumIndex >= 0; + writer.add(fieldNumIndex); + } + } + writer.finish(); + } + + private void flushFlags(int totalFields, int[] fieldNums) throws IOException { + // check if fields always have the same flags + boolean nonChangingFlags = true; + int[] fieldFlags = new int[fieldNums.length]; + Arrays.fill(fieldFlags, -1); + outer: + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum); + assert fieldNumOff >= 0; + if (fieldFlags[fieldNumOff] == -1) { + fieldFlags[fieldNumOff] = fd.flags; + } else if (fieldFlags[fieldNumOff] != fd.flags) { + nonChangingFlags = false; + break outer; + } + } + } + + if (nonChangingFlags) { + // write one flag per field num + vectorsStream.writeVInt(0); + final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldFlags.length, FLAGS_BITS, 1); + for (int flags : fieldFlags) { + assert flags >= 0; + writer.add(flags); + } + assert writer.ord() == fieldFlags.length - 1; + writer.finish(); + } else { + // write one flag for every field instance + vectorsStream.writeVInt(1); + final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, FLAGS_BITS, 1); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + writer.add(fd.flags); + } + } + assert writer.ord() == totalFields - 1; + writer.finish(); + } + } + + private void flushNumTerms(int totalFields) throws IOException { + int maxNumTerms = 0; + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + maxNumTerms |= fd.numTerms; + } + } + final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); + vectorsStream.writeVInt(bitsRequired); + final PackedInts.Writer writer = PackedInts.getWriterNoHeader( + vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + writer.add(fd.numTerms); + } + } + assert writer.ord() == totalFields - 1; + writer.finish(); + } + + private void flushTermLengths() throws IOException { + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + for (int i = 0; i < fd.numTerms; ++i) { + writer.add(fd.prefixLengths[i]); + } + } + } + writer.finish(); + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + for (int i = 0; i < fd.numTerms; ++i) { + writer.add(fd.suffixLengths[i]); + } + } + } + writer.finish(); + } + + private void flushTermFreqs() throws IOException { + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + for (int i = 0; i < fd.numTerms; ++i) { + writer.add(fd.freqs[i] - 1); + } + } + } + writer.finish(); + } + + private void flushPositions() throws IOException { + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + if (fd.hasPositions) { + int pos = 0; + for (int i = 0; i < fd.numTerms; ++i) { + int previousPosition = 0; + for (int j = 0; j < fd.freqs[i]; ++j) { + final int position = positionsBuf[fd .posStart + pos++]; + writer.add(position - previousPosition); + previousPosition = position; + } + } + assert pos == fd.totalPositions; + } + } + } + writer.finish(); + } + + private void flushOffsets(int[] fieldNums) throws IOException { + boolean hasOffsets = false; + long[] sumPos = new long[fieldNums.length]; + long[] sumOffsets = new long[fieldNums.length]; + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + hasOffsets |= fd.hasOffsets; + if (fd.hasOffsets && fd.hasPositions) { + final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum); + int pos = 0; + for (int i = 0; i < fd.numTerms; ++i) { + int previousPos = 0; + int previousOff = 0; + for (int j = 0; j < fd.freqs[i]; ++j) { + final int position = positionsBuf[fd.posStart + pos]; + final int startOffset = startOffsetsBuf[fd.offStart + pos]; + sumPos[fieldNumOff] += position - previousPos; + sumOffsets[fieldNumOff] += startOffset - previousOff; + previousPos = position; + previousOff = startOffset; + ++pos; + } + } + assert pos == fd.totalPositions; + } + } + } + + if (!hasOffsets) { + // nothing to do + return; + } + + final float[] charsPerTerm = new float[fieldNums.length]; + for (int i = 0; i < fieldNums.length; ++i) { + charsPerTerm[i] = (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float) ((double) sumOffsets[i] / sumPos[i]); + } + + // start offsets + for (int i = 0; i < fieldNums.length; ++i) { + vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i])); + } + + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + if ((fd.flags & OFFSETS) != 0) { + final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum); + final float cpt = charsPerTerm[fieldNumOff]; + int pos = 0; + for (int i = 0; i < fd.numTerms; ++i) { + int previousPos = 0; + int previousOff = 0; + for (int j = 0; j < fd.freqs[i]; ++j) { + final int position = fd.hasPositions ? positionsBuf[fd.posStart + pos] : 0; + final int startOffset = startOffsetsBuf[fd.offStart + pos]; + writer.add(startOffset - previousOff - (int) (cpt * (position - previousPos))); + previousPos = position; + previousOff = startOffset; + ++pos; + } + } + } + } + } + writer.finish(); + + // lengths + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + if ((fd.flags & OFFSETS) != 0) { + int pos = 0; + for (int i = 0; i < fd.numTerms; ++i) { + for (int j = 0; j < fd.freqs[i]; ++j) { + writer.add(lengthsBuf[fd.offStart + pos++] - fd.prefixLengths[i] - fd.suffixLengths[i]); + } + } + assert pos == fd.totalPositions; + } + } + } + writer.finish(); + } + + private void flushPayloadLengths() throws IOException { + writer.reset(vectorsStream); + for (DocData dd : pendingDocs) { + for (FieldData fd : dd.fields) { + if (fd.hasPayloads) { + for (int i = 0; i < fd.totalPositions; ++i) { + writer.add(payloadLengthsBuf[fd.payStart + i]); + } + } + } + } + writer.finish(); + } + + @Override + public void finish(FieldInfos fis, int numDocs) throws IOException { + if (!pendingDocs.isEmpty()) { + flush(); + } + if (numDocs != this.numDocs) { + throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs); + } + indexWriter.finish(numDocs); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public void addProx(int numProx, DataInput positions, DataInput offsets) + throws IOException { + assert (curField.hasPositions) == (positions != null); + assert (curField.hasOffsets) == (offsets != null); + + if (curField.hasPositions) { + final int posStart = curField.posStart + curField.totalPositions; + if (posStart + numProx > positionsBuf.length) { + positionsBuf = ArrayUtil.grow(positionsBuf, posStart + numProx); + } + int position = 0; + if (curField.hasPayloads) { + final int payStart = curField.payStart + curField.totalPositions; + if (payStart + numProx > payloadLengthsBuf.length) { + payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf, payStart + numProx); + } + for (int i = 0; i < numProx; ++i) { + final int code = positions.readVInt(); + if ((code & 1) != 0) { + // This position has a payload + final int payloadLength = positions.readVInt(); + payloadLengthsBuf[payStart + i] = payloadLength; + payloadBytes.copyBytes(positions, payloadLength); + } else { + payloadLengthsBuf[payStart + i] = 0; + } + position += code >>> 1; + positionsBuf[posStart + i] = position; + } + } else { + for (int i = 0; i < numProx; ++i) { + position += (positions.readVInt() >>> 1); + positionsBuf[posStart + i] = position; + } + } + } + + if (curField.hasOffsets) { + final int offStart = curField.offStart + curField.totalPositions; + if (offStart + numProx > startOffsetsBuf.length) { + final int newLength = ArrayUtil.oversize(offStart + numProx, 4); + startOffsetsBuf = Arrays.copyOf(startOffsetsBuf, newLength); + lengthsBuf = Arrays.copyOf(lengthsBuf, newLength); + } + int lastOffset = 0, startOffset, endOffset; + for (int i = 0; i < numProx; ++i) { + startOffset = lastOffset + offsets.readVInt(); + endOffset = startOffset + offsets.readVInt(); + lastOffset = endOffset; + startOffsetsBuf[offStart + i] = startOffset; + lengthsBuf[offStart + i] = endOffset - startOffset; + } + } + + curField.totalPositions += numProx; + } + + @Override + public int merge(MergeState mergeState) throws IOException { + int docCount = 0; + int idx = 0; + + for (AtomicReader reader : mergeState.readers) { + final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++]; + CompressingTermVectorsReader matchingVectorsReader = null; + if (matchingSegmentReader != null) { + final TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReader(); + // we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader + if (vectorsReader != null && vectorsReader instanceof CompressingTermVectorsReader) { + matchingVectorsReader = (CompressingTermVectorsReader) vectorsReader; + } + } + + final int maxDoc = reader.maxDoc(); + final Bits liveDocs = reader.getLiveDocs(); + + if (matchingVectorsReader == null + || matchingVectorsReader.getCompressionMode() != compressionMode + || matchingVectorsReader.getChunkSize() != chunkSize + || matchingVectorsReader.getPackedIntsVersion() != PackedInts.VERSION_CURRENT) { + // naive merge... + for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) { + final Fields vectors = reader.getTermVectors(i); + addAllDocVectors(vectors, mergeState); + ++docCount; + mergeState.checkAbort.work(300); + } + } else { + final CompressingStoredFieldsIndexReader index = matchingVectorsReader.getIndex(); + final IndexInput vectorsStream = matchingVectorsReader.getVectorsStream(); + for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; ) { + if (pendingDocs.isEmpty() + && (i == 0 || index.getStartPointer(i - 1) < index.getStartPointer(i))) { // start of a chunk + final long startPointer = index.getStartPointer(i); + vectorsStream.seek(startPointer); + final int docBase = vectorsStream.readVInt(); + final int chunkDocs = vectorsStream.readVInt(); + assert docBase + chunkDocs <= matchingSegmentReader.maxDoc(); + if (docBase + chunkDocs < matchingSegmentReader.maxDoc() + && nextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs) { + final long chunkEnd = index.getStartPointer(docBase + chunkDocs); + final long chunkLength = chunkEnd - vectorsStream.getFilePointer(); + indexWriter.writeIndex(chunkDocs, this.vectorsStream.getFilePointer()); + this.vectorsStream.writeVInt(docCount); + this.vectorsStream.writeVInt(chunkDocs); + this.vectorsStream.copyBytes(vectorsStream, chunkLength); + docCount += chunkDocs; + this.numDocs += chunkDocs; + mergeState.checkAbort.work(300 * chunkDocs); + i = nextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc); + } else { + for (; i < docBase + chunkDocs; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) { + final Fields vectors = reader.getTermVectors(i); + addAllDocVectors(vectors, mergeState); + ++docCount; + mergeState.checkAbort.work(300); + } + } + } else { + final Fields vectors = reader.getTermVectors(i); + addAllDocVectors(vectors, mergeState); + ++docCount; + mergeState.checkAbort.work(300); + i = nextLiveDoc(i + 1, liveDocs, maxDoc); + } + } + } + } + finish(mergeState.fieldInfos, docCount); + return docCount; + } + + private static int nextLiveDoc(int doc, Bits liveDocs, int maxDoc) { + if (liveDocs == null) { + return doc; + } + while (doc < maxDoc && !liveDocs.get(doc)) { + ++doc; + } + return doc; + } + + private static int nextDeletedDoc(int doc, Bits liveDocs, int maxDoc) { + if (liveDocs == null) { + return maxDoc; + } + while (doc < maxDoc && liveDocs.get(doc)) { + ++doc; + } + return doc; + } + +} Modification de propriétés sur lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java (copie de travail) @@ -0,0 +1,102 @@ +package org.apache.lucene.codecs.compressing; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.TermVectorsReader; +import org.apache.lucene.codecs.TermVectorsWriter; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * A {@link TermVectorsFormat} that compresses chunks of documents together in + * order to improve the compression ratio. + * @lucene.experimental + */ +public final class CompressingTermVectorsFormat extends TermVectorsFormat { + + private final String formatName; + private final String segmentSuffix; + private final CompressionMode compressionMode; + private final int chunkSize; + + /** + * Create a new {@link CompressingTermVectorsFormat}. + *

+ * formatName is the name of the format. This name will be used + * in the file formats to perform + * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}. + *

+ * The compressionMode parameter allows you to choose between + * compression algorithms that have various compression and decompression + * speeds so that you can pick the one that best fits your indexing and + * searching throughput. You should never instantiate two + * {@link CompressingTermVectorsFormat}s that have the same name but + * different {@link CompressionMode}s. + *

+ * chunkSize is the minimum byte size of a chunk of documents. + * Higher values of chunkSize should improve the compression + * ratio but will require more memory at indexing time and might make document + * loading a little slower (depending on the size of your OS cache compared + * to the size of your index). + * + * @param formatName the name of the {@link StoredFieldsFormat} + * @param segmentSuffix a suffix to append to files created by this format + * @param compressionMode the {@link CompressionMode} to use + * @param chunkSize the minimum number of bytes of a single chunk of stored documents + * @see CompressionMode + */ + public CompressingTermVectorsFormat(String formatName, String segmentSuffix, + CompressionMode compressionMode, int chunkSize) { + this.formatName = formatName; + this.segmentSuffix = segmentSuffix; + this.compressionMode = compressionMode; + if (chunkSize < 1) { + throw new IllegalArgumentException("chunkSize must be >= 1"); + } + this.chunkSize = chunkSize; + } + + @Override + public TermVectorsReader vectorsReader(Directory directory, + SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) + throws IOException { + return new CompressingTermVectorsReader(directory, segmentInfo, segmentSuffix, + fieldInfos, context, formatName, compressionMode); + } + + @Override + public TermVectorsWriter vectorsWriter(Directory directory, + SegmentInfo segmentInfo, IOContext context) throws IOException { + return new CompressingTermVectorsWriter(directory, segmentInfo, segmentSuffix, + context, formatName, compressionMode, chunkSize); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(compressionMode=" + compressionMode + + ", chunkSize=" + chunkSize + ")"; + } + +} Modification de propriétés sur lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (révision 1435301) +++ lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (copie de travail) @@ -71,11 +71,12 @@ return i; } - final DataInput in; + DataInput in; final int packedIntsVersion; - final long valueCount; + long valueCount; final int blockSize; - final LongsRef values; + final long[] values; + final LongsRef valuesRef; byte[] blocks; int off; long ord; @@ -87,10 +88,17 @@ */ public BlockPackedReader(DataInput in, int packedIntsVersion, int blockSize, long valueCount) { checkBlockSize(blockSize); - this.in = in; this.packedIntsVersion = packedIntsVersion; this.blockSize = blockSize; - this.values = new LongsRef(blockSize); + this.values = new long[blockSize]; + this.valuesRef = new LongsRef(this.values, 0, 0); + reset(in, valueCount); + } + + /** Reset the current reader to wrap a stream of valueCount + * values contained in in. The block size remains unchanged. */ + public void reset(DataInput in, long valueCount) { + this.in = in; assert valueCount >= 0; this.valueCount = valueCount; off = blockSize; @@ -159,9 +167,15 @@ /** Read the next value. */ public long next() throws IOException { - next(1); - assert values.length == 1; - return values.longs[values.offset]; + if (ord == valueCount) { + throw new EOFException(); + } + if (off == blockSize) { + refill(); + } + final long value = values[off++]; + ++ord; + return value; } /** Read between 1 and count values. */ @@ -177,11 +191,11 @@ count = Math.min(count, blockSize - off); count = (int) Math.min(count, valueCount - ord); - values.offset = off; - values.length = count; + valuesRef.offset = off; + valuesRef.length = count; off += count; ord += count; - return values; + return valuesRef; } private void refill() throws IOException { @@ -195,7 +209,7 @@ assert minEquals0 || minValue != 0; if (bitsPerValue == 0) { - Arrays.fill(values.longs, minValue); + Arrays.fill(values, minValue); } else { final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue); final int iterations = blockSize / decoder.valueCount(); @@ -208,11 +222,11 @@ final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue); in.readBytes(blocks, 0, blocksCount); - decoder.decode(blocks, 0, values.longs, 0, iterations); + decoder.decode(blocks, 0, values, 0, iterations); if (minValue != 0) { for (int i = 0; i < valueCount; ++i) { - values.longs[i] += minValue; + values[i] += minValue; } } } Index: lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (révision 1435301) +++ lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (copie de travail) @@ -62,7 +62,7 @@ out.writeByte((byte) i); } - final DataOutput out; + DataOutput out; final long[] values; byte[] blocks; int off; @@ -75,8 +75,14 @@ */ public BlockPackedWriter(DataOutput out, int blockSize) { checkBlockSize(blockSize); + reset(out); + values = new long[blockSize]; + } + + /** Reset this writer to wrap out. The block size remains unchanged. */ + public void reset(DataOutput out) { + assert out != null; this.out = out; - values = new long[blockSize]; off = 0; ord = 0L; finished = false; @@ -99,7 +105,8 @@ } /** Flush all buffered data to disk. This instance is not usable anymore - * after this method has been called. */ + * after this method has been called until {@link #reset(DataOutput)} has + * been called. */ public void finish() throws IOException { checkNotFinished(); if (off > 0) { Index: lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (révision 1435301) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (copie de travail) @@ -21,6 +21,7 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec; import org.apache.lucene.codecs.lucene41.Lucene41Codec; @@ -66,6 +67,7 @@ } private final CompressingStoredFieldsFormat storedFieldsFormat; + private final CompressingTermVectorsFormat termVectorsFormat; /** * Creates a compressing codec with a given segment suffix @@ -73,6 +75,7 @@ public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize) { super(name, new Lucene41Codec()); this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize); + this.termVectorsFormat = new CompressingTermVectorsFormat(name, segmentSuffix, compressionMode, chunkSize); } /** @@ -88,7 +91,12 @@ } @Override + public TermVectorsFormat termVectorsFormat() { + return termVectorsFormat; + } + + @Override public String toString() { - return getName() + "(storedFieldsFormat=" + storedFieldsFormat + ")"; + return getName() + "(storedFieldsFormat=" + storedFieldsFormat + ", termVectorsFormat=" + termVectorsFormat + ")"; } }