diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java 2013-09-01 19:36:22.145292639 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java 2013-08-26 22:50:46.206824737 +0800 @@ -142,6 +142,7 @@ final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); final long sumDocFreq = in.readVLong(); final int docCount = in.readVInt(); + final int longsSize = version >= BlockTermsWriter.VERSION_META_ARRAY ? in.readVInt() : 0; if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")"); } @@ -151,7 +152,7 @@ if (sumTotalTermFreq != -1 && sumTotalTermFreq < sumDocFreq) { // #positions must be >= #postings throw new CorruptIndexException("invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq + " (resource=" + in + ")"); } - FieldReader previous = fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, termsStartPointer, sumTotalTermFreq, sumDocFreq, docCount)); + FieldReader previous = fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, termsStartPointer, sumTotalTermFreq, sumDocFreq, docCount, longsSize)); if (previous != null) { throw new CorruptIndexException("duplicate fields: " + fieldInfo.name + " (resource=" + in + ")"); } @@ -230,8 +231,9 @@ final long sumTotalTermFreq; final long sumDocFreq; final int docCount; + final int longsSize; - FieldReader(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long sumTotalTermFreq, long sumDocFreq, int docCount) { + FieldReader(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) { assert numTerms > 0; this.fieldInfo = fieldInfo; this.numTerms = numTerms; @@ -239,6 +241,7 @@ this.sumTotalTermFreq = sumTotalTermFreq; this.sumDocFreq = sumDocFreq; this.docCount = docCount; + this.longsSize = longsSize; } @Override @@ -326,6 +329,10 @@ private final ByteArrayDataInput freqReader = new ByteArrayDataInput(); private int metaDataUpto; + private long[] longs; + private byte[] bytes; + private ByteArrayDataInput bytesReader; + public SegmentTermsEnum() throws IOException { in = BlockTermsReader.this.in.clone(); in.seek(termsStartPointer); @@ -339,6 +346,7 @@ termSuffixes = new byte[128]; docFreqBytes = new byte[64]; //System.out.println("BTR.enum init this=" + this + " postingsReader=" + postingsReader); + longs = new long[longsSize]; } @Override @@ -415,7 +423,7 @@ assert result; indexIsCurrent = true; - didIndexNext = false; + didIndexNext = false; if (doOrd) { state.ord = indexEnum.ord()-1; @@ -789,11 +797,20 @@ //System.out.println(" freq bytes len=" + len); in.readBytes(docFreqBytes, 0, len); freqReader.reset(docFreqBytes, 0, len); - metaDataUpto = 0; - state.termBlockOrd = 0; + // metadata + len = in.readVInt(); + if (bytes == null) { + bytes = new byte[ArrayUtil.oversize(len, 1)]; + bytesReader = new ByteArrayDataInput(); + } else if (bytes.length < len) { + bytes = new byte[ArrayUtil.oversize(len, 1)]; + } + in.readBytes(bytes, 0, len); + bytesReader.reset(bytes, 0, len); - postingsReader.readTermsBlock(in, fieldInfo, state); + metaDataUpto = 0; + state.termBlockOrd = 0; indexIsCurrent = false; //System.out.println(" indexIsCurrent=" + indexIsCurrent); @@ -811,9 +828,7 @@ // lazily catch up on metadata decode: final int limit = state.termBlockOrd; - // We must set/incr state.termCount because - // postings impl can look at this - state.termBlockOrd = metaDataUpto; + boolean absolute = metaDataUpto == 0; // TODO: better API would be "jump straight to term=N"??? while (metaDataUpto < limit) { //System.out.println(" decode mdUpto=" + metaDataUpto); @@ -825,16 +840,21 @@ // TODO: if docFreq were bulk decoded we could // just skipN here: + + // docFreq, totalTermFreq state.docFreq = freqReader.readVInt(); //System.out.println(" dF=" + state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { state.totalTermFreq = state.docFreq + freqReader.readVLong(); //System.out.println(" totTF=" + state.totalTermFreq); } - - postingsReader.nextTerm(fieldInfo, state); + // metadata + for (int i = 0; i < longs.length; i++) { + longs[i] = bytesReader.readVLong(); + } + postingsReader.decodeTerm(longs, bytesReader, fieldInfo, state, absolute); metaDataUpto++; - state.termBlockOrd++; + absolute = false; } } else { //System.out.println(" skip! seekPending"); diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java 2013-09-01 19:36:22.145292639 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java 2013-09-03 19:43:02.069968383 +0800 @@ -27,6 +27,7 @@ import org.apache.lucene.codecs.PostingsConsumer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; +import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.TermsConsumer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; @@ -59,7 +60,8 @@ // Initial format public static final int VERSION_START = 0; public static final int VERSION_APPEND_ONLY = 1; - public static final int VERSION_CURRENT = VERSION_APPEND_ONLY; + public static final int VERSION_META_ARRAY = 2; + public static final int VERSION_CURRENT = VERSION_META_ARRAY; /** Extension of terms file */ static final String TERMS_EXTENSION = "tib"; @@ -77,8 +79,9 @@ public final long sumTotalTermFreq; public final long sumDocFreq; public final int docCount; + public final int longsSize; - public FieldMetaData(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long sumTotalTermFreq, long sumDocFreq, int docCount) { + public FieldMetaData(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) { assert numTerms > 0; this.fieldInfo = fieldInfo; this.termsStartPointer = termsStartPointer; @@ -86,6 +89,7 @@ this.sumTotalTermFreq = sumTotalTermFreq; this.sumDocFreq = sumDocFreq; this.docCount = docCount; + this.longsSize = longsSize; } } @@ -109,7 +113,7 @@ //System.out.println("BTW.init seg=" + state.segmentName); - postingsWriter.start(out); // have consumer write its format/header + postingsWriter.init(out); // have consumer write its format/header success = true; } finally { if (!success) { @@ -133,9 +137,7 @@ @Override public void close() throws IOException { - try { - final long dirStart = out.getFilePointer(); out.writeVInt(fields.size()); @@ -148,6 +150,9 @@ } out.writeVLong(field.sumDocFreq); out.writeVInt(field.docCount); + if (VERSION_CURRENT >= VERSION_META_ARRAY) { + out.writeVInt(field.longsSize); + } } writeTrailer(dirStart); } finally { @@ -161,7 +166,7 @@ private static class TermEntry { public final BytesRef term = new BytesRef(); - public TermStats stats; + public BlockTermState state; } class TermsWriter extends TermsConsumer { @@ -173,6 +178,7 @@ long sumTotalTermFreq; long sumDocFreq; int docCount; + int longsSize; private TermEntry[] pendingTerms; @@ -190,8 +196,8 @@ pendingTerms[i] = new TermEntry(); } termsStartPointer = out.getFilePointer(); - postingsWriter.setField(fieldInfo); this.postingsWriter = postingsWriter; + this.longsSize = postingsWriter.setField(fieldInfo); } @Override @@ -237,11 +243,12 @@ } final TermEntry te = pendingTerms[pendingCount]; te.term.copyBytes(text); - te.stats = stats; + te.state = postingsWriter.newTermState(); + te.state.docFreq = stats.docFreq; + te.state.totalTermFreq = stats.totalTermFreq; + postingsWriter.finishTerm(te.state); pendingCount++; - - postingsWriter.finishTerm(stats); numTerms++; } @@ -264,7 +271,8 @@ termsStartPointer, sumTotalTermFreq, sumDocFreq, - docCount)); + docCount, + longsSize)); } } @@ -285,6 +293,7 @@ } private final RAMOutputStream bytesWriter = new RAMOutputStream(); + private final RAMOutputStream bufferWriter = new RAMOutputStream(); private void flushBlock() throws IOException { //System.out.println("BTW.flushBlock seg=" + segment + " pendingCount=" + pendingCount + " fp=" + out.getFilePointer()); @@ -318,19 +327,34 @@ // TODO: cutover to better intblock codec. simple64? // write prefix, suffix first: for(int termCount=0;termCount= 0; if (absolute) { indexOut.writeVInt(upto); diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java 2013-09-01 19:36:22.131959306 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java 2013-08-23 23:36:52.326439097 +0800 @@ -79,7 +79,7 @@ // Terms that have <= freqCutoff number of docs are // "pulsed" (inlined): - pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); + pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter); FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, minBlockSize, maxBlockSize); success = true; return ret; @@ -98,7 +98,7 @@ boolean success = false; try { docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); - pulsingReader = new PulsingPostingsReader(docsReader); + pulsingReader = new PulsingPostingsReader(state, docsReader); FieldsProducer ret = new BlockTreeTermsReader( state.directory, state.fieldInfos, state.segmentInfo, pulsingReader, diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java 2013-09-01 19:36:22.128625973 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java 2013-08-31 05:53:47.743391713 +0800 @@ -20,16 +20,20 @@ import java.io.IOException; import java.util.IdentityHashMap; import java.util.Map; +import java.util.TreeMap; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Attribute; @@ -37,6 +41,7 @@ import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; /** Concrete class that reads the current doc/freq/skip * postings format @@ -50,28 +55,53 @@ // Fallback reader for non-pulsed terms: final PostingsReaderBase wrappedPostingsReader; + final SegmentReadState segmentState; int maxPositions; + int version; + TreeMap fields; - public PulsingPostingsReader(PostingsReaderBase wrappedPostingsReader) { + public PulsingPostingsReader(SegmentReadState state, PostingsReaderBase wrappedPostingsReader) { this.wrappedPostingsReader = wrappedPostingsReader; + this.segmentState = state; } @Override public void init(IndexInput termsIn) throws IOException { - CodecUtil.checkHeader(termsIn, PulsingPostingsWriter.CODEC, - PulsingPostingsWriter.VERSION_START, PulsingPostingsWriter.VERSION_START); + version = CodecUtil.checkHeader(termsIn, PulsingPostingsWriter.CODEC, + PulsingPostingsWriter.VERSION_START, + PulsingPostingsWriter.VERSION_CURRENT); maxPositions = termsIn.readVInt(); wrappedPostingsReader.init(termsIn); + if (wrappedPostingsReader instanceof PulsingPostingsReader || + version < PulsingPostingsWriter.VERSION_META_ARRAY) { + fields = null; + } else { + fields = new TreeMap(); + String summaryFileName = IndexFileNames.segmentFileName(segmentState.segmentInfo.name, segmentState.segmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION); + IndexInput in = null; + try { + in = segmentState.directory.openInput(summaryFileName, segmentState.context); + CodecUtil.checkHeader(in, PulsingPostingsWriter.CODEC, version, + PulsingPostingsWriter.VERSION_CURRENT); + int numField = in.readVInt(); + for (int i = 0; i < numField; i++) { + int fieldNum = in.readVInt(); + int longsSize = in.readVInt(); + fields.put(fieldNum, longsSize); + } + } finally { + IOUtils.closeWhileHandlingException(in); + } + } } private static class PulsingTermState extends BlockTermState { + private boolean absolute = false; + private long[] longs; private byte[] postings; private int postingsSize; // -1 if this term was not inlined private BlockTermState wrappedTermState; - ByteArrayDataInput inlinedBytesReader; - private byte[] inlinedBytes; - @Override public PulsingTermState clone() { PulsingTermState clone; @@ -82,6 +112,11 @@ } else { assert wrappedTermState != null; clone.wrappedTermState = (BlockTermState) wrappedTermState.clone(); + clone.absolute = absolute; + if (longs != null) { + clone.longs = new long[longs.length]; + System.arraycopy(longs, 0, clone.longs, 0, longs.length); + } } return clone; } @@ -99,11 +134,6 @@ } else { wrappedTermState.copyFrom(other.wrappedTermState); } - - // NOTE: we do not copy the - // inlinedBytes/inlinedBytesReader; these are only - // stored on the "primary" TermState. They are - // "transient" to cloned term states. } @Override @@ -117,25 +147,6 @@ } @Override - public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException { - //System.out.println("PR.readTermsBlock state=" + _termState); - final PulsingTermState termState = (PulsingTermState) _termState; - if (termState.inlinedBytes == null) { - termState.inlinedBytes = new byte[128]; - termState.inlinedBytesReader = new ByteArrayDataInput(); - } - int len = termsIn.readVInt(); - //System.out.println(" len=" + len + " fp=" + termsIn.getFilePointer()); - if (termState.inlinedBytes.length < len) { - termState.inlinedBytes = new byte[ArrayUtil.oversize(len, 1)]; - } - termsIn.readBytes(termState.inlinedBytes, 0, len); - termState.inlinedBytesReader.reset(termState.inlinedBytes); - termState.wrappedTermState.termBlockOrd = 0; - wrappedPostingsReader.readTermsBlock(termsIn, fieldInfo, termState.wrappedTermState); - } - - @Override public BlockTermState newTermState() throws IOException { PulsingTermState state = new PulsingTermState(); state.wrappedTermState = wrappedPostingsReader.newTermState(); @@ -143,20 +154,20 @@ } @Override - public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) throws IOException { + public void decodeTerm(long[] empty, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute) throws IOException { //System.out.println("PR nextTerm"); PulsingTermState termState = (PulsingTermState) _termState; - + assert empty.length == 0; + termState.absolute = termState.absolute || absolute; // if we have positions, its total TF, otherwise its computed based on docFreq. long count = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 ? termState.totalTermFreq : termState.docFreq; //System.out.println(" count=" + count + " threshold=" + maxPositions); if (count <= maxPositions) { - // Inlined into terms dict -- just read the byte[] blob in, // but don't decode it now (we only decode when a DocsEnum // or D&PEnum is pulled): - termState.postingsSize = termState.inlinedBytesReader.readVInt(); + termState.postingsSize = in.readVInt(); if (termState.postings == null || termState.postings.length < termState.postingsSize) { termState.postings = new byte[ArrayUtil.oversize(termState.postingsSize, 1)]; } @@ -164,16 +175,23 @@ // (the blob holding all inlined terms' blobs for // current term block) into another byte[] (just the // blob for this term)... - termState.inlinedBytesReader.readBytes(termState.postings, 0, termState.postingsSize); + in.readBytes(termState.postings, 0, termState.postingsSize); //System.out.println(" inlined bytes=" + termState.postingsSize); + termState.absolute = termState.absolute || absolute; } else { //System.out.println(" not inlined"); + final int longsSize = fields == null ? 0 : fields.get(fieldInfo.number); + if (termState.longs == null) { + termState.longs = new long[longsSize]; + } + for (int i = 0; i < longsSize; i++) { + termState.longs[i] = in.readVLong(); + } termState.postingsSize = -1; - // TODO: should we do full copyFrom? much heavier...? termState.wrappedTermState.docFreq = termState.docFreq; termState.wrappedTermState.totalTermFreq = termState.totalTermFreq; - wrappedPostingsReader.nextTerm(fieldInfo, termState.wrappedTermState); - termState.wrappedTermState.termBlockOrd++; + wrappedPostingsReader.decodeTerm(termState.longs, in, fieldInfo, termState.wrappedTermState, termState.absolute); + termState.absolute = false; } } diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java 2013-09-01 19:36:22.128625973 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java 2013-09-03 19:43:02.036635049 +0800 @@ -21,14 +21,19 @@ import java.util.List; import java.util.ArrayList; +import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; // TODO: we now inline based on total TF of the term, // but it might be better to inline by "net bytes used" @@ -49,26 +54,43 @@ final static String CODEC = "PulsedPostingsWriter"; + // recording field summary + final static String SUMMARY_EXTENSION = "smy"; + // To add a new version, increment from the last one, and // change VERSION_CURRENT to point to your new version: final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; + final static int VERSION_META_ARRAY = 1; + + final static int VERSION_CURRENT = VERSION_META_ARRAY; + private SegmentWriteState segmentState; private IndexOutput termsOut; + private List fields; + private IndexOptions indexOptions; private boolean storePayloads; - private static class PendingTerm { - private final byte[] bytes; - public PendingTerm(byte[] bytes) { - this.bytes = bytes; + // information for wrapped PF, in current field + private int longsSize; + private long[] longs; + boolean absolute; + + private static class PulsingTermState extends BlockTermState { + private byte[] bytes; + private BlockTermState wrappedState; + @Override + public String toString() { + if (bytes != null) { + return "inlined"; + } else { + return "not inlined wrapped=" + wrappedState; + } } } - private final List pendingTerms = new ArrayList(); - // one entry per position private final Position[] pending; private int pendingCount = 0; // -1 once we've hit too many positions @@ -83,6 +105,15 @@ int endOffset; } + private static final class FieldMetaData { + int fieldNumber; + int longsSize; + FieldMetaData(int number, int size) { + fieldNumber = number; + longsSize = size; + } + } + // TODO: -- lazy init this? ie, if every single term // was inlined (eg for a "primary key" field) then we // never need to use this fallback? Fallback writer for @@ -92,23 +123,33 @@ /** If the total number of positions (summed across all docs * for this term) is <= maxPositions, then the postings are * inlined into terms dict */ - public PulsingPostingsWriter(int maxPositions, PostingsWriterBase wrappedPostingsWriter) { + public PulsingPostingsWriter(SegmentWriteState state, int maxPositions, PostingsWriterBase wrappedPostingsWriter) { + pending = new Position[maxPositions]; for(int i=0;i(); // We simply wrap another postings writer, but only call // on it when tot positions is >= the cutoff: this.wrappedPostingsWriter = wrappedPostingsWriter; + this.segmentState = state; } @Override - public void start(IndexOutput termsOut) throws IOException { + public void init(IndexOutput termsOut) throws IOException { this.termsOut = termsOut; CodecUtil.writeHeader(termsOut, CODEC, VERSION_CURRENT); termsOut.writeVInt(pending.length); // encode maxPositions in header - wrappedPostingsWriter.start(termsOut); + wrappedPostingsWriter.init(termsOut); + } + + @Override + public BlockTermState newTermState() throws IOException { + PulsingTermState state = new PulsingTermState(); + state.wrappedState = wrappedPostingsWriter.newTermState(); + return state; } @Override @@ -123,11 +164,15 @@ // Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes @Override - public void setField(FieldInfo fieldInfo) { + public int setField(FieldInfo fieldInfo) { this.indexOptions = fieldInfo.getIndexOptions(); //if (DEBUG) System.out.println("PW field=" + fieldInfo.name + " indexOptions=" + indexOptions); storePayloads = fieldInfo.hasPayloads(); - wrappedPostingsWriter.setField(fieldInfo); + absolute = false; + longsSize = wrappedPostingsWriter.setField(fieldInfo); + longs = new long[longsSize]; + fields.add(new FieldMetaData(fieldInfo.number, longsSize)); + return 0; //DEBUG = BlockTreeTermsWriter.DEBUG; } @@ -219,18 +264,19 @@ /** Called when we are done adding docs to this term */ @Override - public void finishTerm(TermStats stats) throws IOException { + public void finishTerm(BlockTermState _state) throws IOException { + PulsingTermState state = (PulsingTermState) _state; + // if (DEBUG) System.out.println("PW finishTerm docCount=" + stats.docFreq + " pendingCount=" + pendingCount + " pendingTerms.size()=" + pendingTerms.size()); assert pendingCount > 0 || pendingCount == -1; if (pendingCount == -1) { - wrappedPostingsWriter.finishTerm(stats); - // Must add null entry to record terms that our - // wrapped postings impl added - pendingTerms.add(null); + state.wrappedState.docFreq = state.docFreq; + state.wrappedState.totalTermFreq = state.totalTermFreq; + state.bytes = null; + wrappedPostingsWriter.finishTerm(state.wrappedState); } else { - // There were few enough total occurrences for this // term, so we fully inline our postings data into // terms dict, now: @@ -325,61 +371,54 @@ } } - final byte[] bytes = new byte[(int) buffer.getFilePointer()]; - buffer.writeTo(bytes, 0); - pendingTerms.add(new PendingTerm(bytes)); + state.bytes = new byte[(int) buffer.getFilePointer()]; + buffer.writeTo(state.bytes, 0); buffer.reset(); } - pendingCount = 0; } @Override - public void close() throws IOException { - wrappedPostingsWriter.close(); - } - - @Override - public void flushTermsBlock(int start, int count) throws IOException { - // if (DEBUG) System.out.println("PW: flushTermsBlock start=" + start + " count=" + count + " pendingTerms.size()=" + pendingTerms.size()); - int wrappedCount = 0; - assert buffer.getFilePointer() == 0; - assert start >= count; - - final int limit = pendingTerms.size() - start + count; - - for(int idx=pendingTerms.size()-start; idx + * @lucene.experimental + */ + +public final class TempFSTOrdPostingsFormat extends PostingsFormat { + public TempFSTOrdPostingsFormat() { + super("TempFSTOrd"); + } + + @Override + public String toString() { + return getName(); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state); + + boolean success = false; + try { + FieldsConsumer ret = new TempFSTOrdTermsWriter(state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsWriter); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, + state.fieldInfos, + state.segmentInfo, + state.context, + state.segmentSuffix); + boolean success = false; + try { + FieldsProducer ret = new TempFSTOrdTermsReader(state, postingsReader); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsReader); + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPulsing41PostingsFormat.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPulsing41PostingsFormat.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPulsing41PostingsFormat.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdPulsing41PostingsFormat.java 2013-09-01 10:08:11.755684506 +0800 @@ -0,0 +1,91 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsBaseFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.PostingsWriterBase; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsBaseFormat; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; +import org.apache.lucene.codecs.pulsing.PulsingPostingsWriter; +import org.apache.lucene.codecs.pulsing.PulsingPostingsReader; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; + +/** TempFSTOrd + Pulsing41 + * @lucene.experimental */ + +public class TempFSTOrdPulsing41PostingsFormat extends PostingsFormat { + private final PostingsBaseFormat wrappedPostingsBaseFormat; + private final int freqCutoff; + + public TempFSTOrdPulsing41PostingsFormat() { + this(1); + } + + public TempFSTOrdPulsing41PostingsFormat(int freqCutoff) { + super("TempFSTOrdPulsing41"); + this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat(); + this.freqCutoff = freqCutoff; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = null; + PostingsWriterBase pulsingWriter = null; + + boolean success = false; + try { + docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter); + FieldsConsumer ret = new TempFSTOrdTermsWriter(state, pulsingWriter); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase docsReader = null; + PostingsReaderBase pulsingReader = null; + boolean success = false; + try { + docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + pulsingReader = new PulsingPostingsReader(state, docsReader); + FieldsProducer ret = new TempFSTOrdTermsReader(state, pulsingReader); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(docsReader, pulsingReader); + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsReader.java 2013-09-02 00:34:44.695086852 +0800 @@ -0,0 +1,821 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.File; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeMap; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.fst.BytesRefFSTEnum; +import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.fst.PositiveIntOutputs; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.CodecUtil; + +/** + * FST-based terms dictionary reader. + * + * The FST index maps each term and its ord, and during seek + * the ord is used fetch metadata from a single block. + * The term dictionary is fully memeory resident. + * + * @lucene.experimental + */ +public class TempFSTOrdTermsReader extends FieldsProducer { + static final int INTERVAL = TempFSTOrdTermsWriter.SKIP_INTERVAL; + final TreeMap fields = new TreeMap(); + final PostingsReaderBase postingsReader; + IndexInput indexIn = null; + IndexInput blockIn = null; + //static final boolean TEST = false; + + public TempFSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempFSTOrdTermsWriter.TERMS_INDEX_EXTENSION); + final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempFSTOrdTermsWriter.TERMS_BLOCK_EXTENSION); + + this.postingsReader = postingsReader; + try { + this.indexIn = state.directory.openInput(termsIndexFileName, state.context); + this.blockIn = state.directory.openInput(termsBlockFileName, state.context); + readHeader(indexIn); + readHeader(blockIn); + this.postingsReader.init(blockIn); + seekDir(indexIn); + seekDir(blockIn); + + final FieldInfos fieldInfos = state.fieldInfos; + final int numFields = blockIn.readVInt(); + for (int i = 0; i < numFields; i++) { + FieldInfo fieldInfo = fieldInfos.fieldInfo(blockIn.readVInt()); + boolean hasFreq = fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY; + long numTerms = blockIn.readVLong(); + long sumTotalTermFreq = hasFreq ? blockIn.readVLong() : -1; + long sumDocFreq = blockIn.readVLong(); + int docCount = blockIn.readVInt(); + int longsSize = blockIn.readVInt(); + FST index = new FST(indexIn, PositiveIntOutputs.getSingleton()); + + TermsReader current = new TermsReader(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index); + TermsReader previous = fields.put(fieldInfo.name, current); + checkFieldSummary(state.segmentInfo, current, previous); + } + } finally { + IOUtils.closeWhileHandlingException(indexIn, blockIn); + } + } + + private int readHeader(IndexInput in) throws IOException { + return CodecUtil.checkHeader(in, TempFSTOrdTermsWriter.TERMS_CODEC_NAME, + TempFSTOrdTermsWriter.TERMS_VERSION_START, + TempFSTOrdTermsWriter.TERMS_VERSION_CURRENT); + } + private void seekDir(IndexInput in) throws IOException { + in.seek(in.length() - 8); + in.seek(in.readLong()); + } + private void checkFieldSummary(SegmentInfo info, TermsReader field, TermsReader previous) throws IOException { + // #docs with field must be <= #docs + if (field.docCount < 0 || field.docCount > info.getDocCount()) { + throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + indexIn + ", " + blockIn + ")"); + } + // #postings must be >= #docs with field + if (field.sumDocFreq < field.docCount) { + throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + indexIn + ", " + blockIn + ")"); + } + // #positions must be >= #postings + if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) { + throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + indexIn + ", " + blockIn + ")"); + } + if (previous != null) { + throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + indexIn + ", " + blockIn + ")"); + } + } + + @Override + public Iterator iterator() { + return Collections.unmodifiableSet(fields.keySet()).iterator(); + } + + @Override + public Terms terms(String field) throws IOException { + assert field != null; + return fields.get(field); + } + + @Override + public int size() { + return fields.size(); + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(postingsReader); + } finally { + fields.clear(); + } + } + + final class TermsReader extends Terms { + final FieldInfo fieldInfo; + final long numTerms; + final long sumTotalTermFreq; + final long sumDocFreq; + final int docCount; + final int longsSize; + final FST index; + + final int numSkipInfo; + final long[] skipInfo; + final byte[] statsBlock; + final byte[] metaLongsBlock; + final byte[] metaBytesBlock; + + TermsReader(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST index) throws IOException { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.sumTotalTermFreq = sumTotalTermFreq; + this.sumDocFreq = sumDocFreq; + this.docCount = docCount; + this.longsSize = longsSize; + this.index = index; + + assert (numTerms & (~0xffffffffL)) == 0; + final int numBlocks = (int)(numTerms + INTERVAL - 1) / INTERVAL; + this.numSkipInfo = longsSize + 3; + this.skipInfo = new long[numBlocks * numSkipInfo]; + this.statsBlock = new byte[(int)blockIn.readVLong()]; + this.metaLongsBlock = new byte[(int)blockIn.readVLong()]; + this.metaBytesBlock = new byte[(int)blockIn.readVLong()]; + + int last = 0, next = 0; + for (int i = 1; i < numBlocks; i++) { + next = numSkipInfo * i; + for (int j = 0; j < numSkipInfo; j++) { + skipInfo[next + j] = skipInfo[last + j] + blockIn.readVLong(); + } + last = next; + } + blockIn.readBytes(statsBlock, 0, statsBlock.length); + blockIn.readBytes(metaLongsBlock, 0, metaLongsBlock.length); + blockIn.readBytes(metaBytesBlock, 0, metaBytesBlock.length); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override + public boolean hasOffsets() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + } + + @Override + public boolean hasPositions() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + } + + @Override + public boolean hasPayloads() { + return fieldInfo.hasPayloads(); + } + + @Override + public long size() { + return numTerms; + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public TermsEnum iterator(TermsEnum reuse) throws IOException { + return new SegmentTermsEnum(); + } + + @Override + public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + return new IntersectTermsEnum(compiled, startTerm); + } + + // Only wraps common operations for PBF interact + abstract class BaseTermsEnum extends TermsEnum { + /* Current term, null when enum ends or unpositioned */ + BytesRef term; + + /* Current term's ord, starts from 0 */ + long ord; + + /* Current term stats + decoded metadata (customized by PBF) */ + final BlockTermState state; + + /* Datainput to load stats & metadata */ + final ByteArrayDataInput statsReader = new ByteArrayDataInput(); + final ByteArrayDataInput metaLongsReader = new ByteArrayDataInput(); + final ByteArrayDataInput metaBytesReader = new ByteArrayDataInput(); + + /* To which block is buffered */ + int statsBlockOrd; + int metaBlockOrd; + + /* Current buffered metadata (long[] & byte[]) */ + long[][] longs; + int[] bytesStart; + int[] bytesLength; + + /* Current buffered stats (df & ttf) */ + int[] docFreq; + long[] totalTermFreq; + + BaseTermsEnum() throws IOException { + this.state = postingsReader.newTermState(); + this.term = null; + this.statsReader.reset(statsBlock); + this.metaLongsReader.reset(metaLongsBlock); + this.metaBytesReader.reset(metaBytesBlock); + + this.longs = new long[INTERVAL][longsSize]; + this.bytesStart = new int[INTERVAL]; + this.bytesLength = new int[INTERVAL]; + this.docFreq = new int[INTERVAL]; + this.totalTermFreq = new long[INTERVAL]; + this.statsBlockOrd = -1; + this.metaBlockOrd = -1; + if (!hasFreqs()) { + Arrays.fill(totalTermFreq, -1); + } + } + + /** Decodes stats data into term state */ + void decodeStats() throws IOException { + final int upto = (int)ord % INTERVAL; + final int oldBlockOrd = statsBlockOrd; + statsBlockOrd = (int)ord / INTERVAL; + if (oldBlockOrd != statsBlockOrd) { + refillStats(); + } + state.docFreq = docFreq[upto]; + state.totalTermFreq = totalTermFreq[upto]; + } + + /** Let PBF decode metadata */ + void decodeMetaData() throws IOException { + final int upto = (int)ord % INTERVAL; + final int oldBlockOrd = metaBlockOrd; + metaBlockOrd = (int)ord / INTERVAL; + if (metaBlockOrd != oldBlockOrd) { + refillMetadata(); + } + metaBytesReader.setPosition(bytesStart[upto]); + postingsReader.decodeTerm(longs[upto], metaBytesReader, fieldInfo, state, true); + } + + /** Load current stats shard */ + final void refillStats() throws IOException { + final int offset = statsBlockOrd * numSkipInfo; + final int statsFP = (int)skipInfo[offset]; + statsReader.setPosition(statsFP); + for (int i = 0; i < INTERVAL && !statsReader.eof(); i++) { + int code = statsReader.readVInt(); + if (hasFreqs()) { + docFreq[i] = (code >>> 1); + if ((code & 1) == 1) { + totalTermFreq[i] = docFreq[i]; + } else { + totalTermFreq[i] = docFreq[i] + statsReader.readVLong(); + } + } else { + docFreq[i] = code; + } + } + } + + /** Load current metadata shard */ + final void refillMetadata() throws IOException { + final int offset = metaBlockOrd * numSkipInfo; + final int metaLongsFP = (int)skipInfo[offset + 1]; + final int metaBytesFP = (int)skipInfo[offset + 2]; + metaLongsReader.setPosition(metaLongsFP); + for (int j = 0; j < longsSize; j++) { + longs[0][j] = skipInfo[offset + 3 + j] + metaLongsReader.readVLong(); + } + bytesStart[0] = metaBytesFP; + bytesLength[0] = (int)metaLongsReader.readVLong(); + for (int i = 1; i < INTERVAL && !metaLongsReader.eof(); i++) { + for (int j = 0; j < longsSize; j++) { + longs[i][j] = longs[i-1][j] + metaLongsReader.readVLong(); + } + bytesStart[i] = bytesStart[i-1] + bytesLength[i-1]; + bytesLength[i] = (int)metaLongsReader.readVLong(); + } + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public TermState termState() throws IOException { + decodeMetaData(); + return state.clone(); + } + + @Override + public BytesRef term() { + return term; + } + + @Override + public int docFreq() throws IOException { + return state.docFreq; + } + + @Override + public long totalTermFreq() throws IOException { + return state.totalTermFreq; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + decodeMetaData(); + return postingsReader.docs(fieldInfo, state, liveDocs, reuse, flags); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + if (!hasPositions()) { + return null; + } + decodeMetaData(); + return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); + } + + // TODO: this can be achieved by making use of Util.getByOutput() + // and should have related tests + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + } + + // Iterates through all terms in this field + private final class SegmentTermsEnum extends BaseTermsEnum { + final BytesRefFSTEnum fstEnum; + + /* True when current term's metadata is decoded */ + boolean decoded; + + /* True when current enum is 'positioned' by seekExact(TermState) */ + boolean seekPending; + + SegmentTermsEnum() throws IOException { + this.fstEnum = new BytesRefFSTEnum(index); + this.decoded = false; + this.seekPending = false; + } + + @Override + void decodeMetaData() throws IOException { + if (!decoded && !seekPending) { + super.decodeMetaData(); + decoded = true; + } + } + + // Update current enum according to FSTEnum + void updateEnum(final InputOutput pair) throws IOException { + if (pair == null) { + term = null; + } else { + term = pair.input; + ord = pair.output; + decodeStats(); + } + decoded = false; + seekPending = false; + } + + @Override + public BytesRef next() throws IOException { + if (seekPending) { // previously positioned, but termOutputs not fetched + seekPending = false; + SeekStatus status = seekCeil(term); + assert status == SeekStatus.FOUND; // must positioned on valid term + } + updateEnum(fstEnum.next()); + return term; + } + + @Override + public boolean seekExact(BytesRef target) throws IOException { + updateEnum(fstEnum.seekExact(target)); + return term != null; + } + + @Override + public SeekStatus seekCeil(BytesRef target) throws IOException { + updateEnum(fstEnum.seekCeil(target)); + if (term == null) { + return SeekStatus.END; + } else { + return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND; + } + } + + @Override + public void seekExact(BytesRef target, TermState otherState) { + if (!target.equals(term)) { + state.copyFrom(otherState); + term = BytesRef.deepCopyOf(target); + seekPending = true; + } + } + } + + // Iterates intersect result with automaton (cannot seek!) + private final class IntersectTermsEnum extends BaseTermsEnum { + /* True when current term's metadata is decoded */ + boolean decoded; + + /* True when there is pending term when calling next() */ + boolean pending; + + /* stack to record how current term is constructed, + * used to accumulate metadata or rewind term: + * level == term.length + 1, + * == 0 when term is null */ + Frame[] stack; + int level; + + /* term dict fst */ + final FST fst; + final FST.BytesReader fstReader; + final Outputs fstOutputs; + + /* query automaton to intersect with */ + final ByteRunAutomaton fsa; + + private final class Frame { + /* fst stats */ + FST.Arc arc; + + /* automaton stats */ + int state; + + Frame() { + this.arc = new FST.Arc(); + this.state = -1; + } + + public String toString() { + return "arc=" + arc + " state=" + state; + } + } + + IntersectTermsEnum(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + //if (TEST) System.out.println("Enum init, startTerm=" + startTerm); + this.fst = index; + this.fstReader = fst.getBytesReader(); + this.fstOutputs = index.outputs; + this.fsa = compiled.runAutomaton; + this.level = -1; + this.stack = new Frame[16]; + for (int i = 0 ; i < stack.length; i++) { + this.stack[i] = new Frame(); + } + + Frame frame; + frame = loadVirtualFrame(newFrame()); + this.level++; + frame = loadFirstFrame(newFrame()); + pushFrame(frame); + + this.decoded = false; + this.pending = false; + + if (startTerm == null) { + pending = isAccept(topFrame()); + } else { + doSeekCeil(startTerm); + pending = !startTerm.equals(term) && isValid(topFrame()) && isAccept(topFrame()); + } + } + + @Override + void decodeMetaData() throws IOException { + if (!decoded) { + super.decodeMetaData(); + decoded = true; + } + } + + @Override + void decodeStats() throws IOException { + final FST.Arc arc = topFrame().arc; + assert arc.nextFinalOutput == fstOutputs.getNoOutput(); + ord = arc.output; + super.decodeStats(); + } + + @Override + public SeekStatus seekCeil(BytesRef target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef next() throws IOException { + //if (TEST) System.out.println("Enum next()"); + if (pending) { + pending = false; + decodeStats(); + return term; + } + decoded = false; + DFS: + while (level > 0) { + Frame frame = newFrame(); + if (loadExpandFrame(topFrame(), frame) != null) { // has valid target + pushFrame(frame); + if (isAccept(frame)) { // gotcha + break; + } + continue; // check next target + } + frame = popFrame(); + while(level > 0) { + if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling + pushFrame(frame); + if (isAccept(frame)) { // gotcha + break DFS; + } + continue DFS; // check next target + } + frame = popFrame(); + } + return null; + } + decodeStats(); + return term; + } + + BytesRef doSeekCeil(BytesRef target) throws IOException { + //if (TEST) System.out.println("Enum doSeekCeil()"); + Frame frame= null; + int label, upto = 0, limit = target.length; + while (upto < limit) { // to target prefix, or ceil label (rewind prefix) + frame = newFrame(); + label = target.bytes[upto] & 0xff; + frame = loadCeilFrame(label, topFrame(), frame); + if (frame == null || frame.arc.label != label) { + break; + } + assert isValid(frame); // target must be fetched from automaton + pushFrame(frame); + upto++; + } + if (upto == limit) { // got target + return term; + } + if (frame != null) { // got larger term('s prefix) + pushFrame(frame); + return isAccept(frame) ? term : next(); + } + while (level > 0) { // got target's prefix, advance to larger term + frame = popFrame(); + while (level > 0 && !canRewind(frame)) { + frame = popFrame(); + } + if (loadNextFrame(topFrame(), frame) != null) { + pushFrame(frame); + return isAccept(frame) ? term : next(); + } + } + return null; + } + + /** Virtual frame, never pop */ + Frame loadVirtualFrame(Frame frame) throws IOException { + frame.arc.output = fstOutputs.getNoOutput(); + frame.arc.nextFinalOutput = fstOutputs.getNoOutput(); + frame.state = -1; + return frame; + } + + /** Load frame for start arc(node) on fst */ + Frame loadFirstFrame(Frame frame) throws IOException { + frame.arc = fst.getFirstArc(frame.arc); + frame.state = fsa.getInitialState(); + return frame; + } + + /** Load frame for target arc(node) on fst */ + Frame loadExpandFrame(Frame top, Frame frame) throws IOException { + if (!canGrow(top)) { + return null; + } + frame.arc = fst.readFirstRealTargetArc(top.arc.target, frame.arc, fstReader); + frame.state = fsa.step(top.state, frame.arc.label); + //if (TEST) System.out.println(" loadExpand frame="+frame); + if (frame.state == -1) { + return loadNextFrame(top, frame); + } + return frame; + } + + /** Load frame for sibling arc(node) on fst */ + Frame loadNextFrame(Frame top, Frame frame) throws IOException { + if (!canRewind(frame)) { + return null; + } + while (!frame.arc.isLast()) { + frame.arc = fst.readNextRealArc(frame.arc, fstReader); + frame.state = fsa.step(top.state, frame.arc.label); + if (frame.state != -1) { + break; + } + } + //if (TEST) System.out.println(" loadNext frame="+frame); + if (frame.state == -1) { + return null; + } + return frame; + } + + /** Load frame for target arc(node) on fst, so that + * arc.label >= label and !fsa.reject(arc.label) */ + Frame loadCeilFrame(int label, Frame top, Frame frame) throws IOException { + FST.Arc arc = frame.arc; + arc = Util.readCeilArc(label, fst, top.arc, arc, fstReader); + if (arc == null) { + return null; + } + frame.state = fsa.step(top.state, arc.label); + //if (TEST) System.out.println(" loadCeil frame="+frame); + if (frame.state == -1) { + return loadNextFrame(top, frame); + } + return frame; + } + + boolean isAccept(Frame frame) { // reach a term both fst&fsa accepts + return fsa.isAccept(frame.state) && frame.arc.isFinal(); + } + boolean isValid(Frame frame) { // reach a prefix both fst&fsa won't reject + return /*frame != null &&*/ frame.state != -1; + } + boolean canGrow(Frame frame) { // can walk forward on both fst&fsa + return frame.state != -1 && FST.targetHasArcs(frame.arc); + } + boolean canRewind(Frame frame) { // can jump to sibling + return !frame.arc.isLast(); + } + + void pushFrame(Frame frame) { + final FST.Arc arc = frame.arc; + arc.output = fstOutputs.add(topFrame().arc.output, arc.output); + term = grow(arc.label); + level++; + assert frame == stack[level]; + } + + Frame popFrame() { + term = shrink(); + return stack[level--]; + } + + Frame newFrame() { + if (level+1 == stack.length) { + final Frame[] temp = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + System.arraycopy(stack, 0, temp, 0, stack.length); + for (int i = stack.length; i < temp.length; i++) { + temp[i] = new Frame(); + } + stack = temp; + } + return stack[level+1]; + } + + Frame topFrame() { + return stack[level]; + } + + BytesRef grow(int label) { + if (term == null) { + term = new BytesRef(new byte[16], 0, 0); + } else { + if (term.length == term.bytes.length) { + term.grow(term.length+1); + } + term.bytes[term.length++] = (byte)label; + } + return term; + } + + BytesRef shrink() { + if (term.length == 0) { + term = null; + } else { + term.length--; + } + return term; + } + } + } + + static void walk(FST fst) throws IOException { + final ArrayList> queue = new ArrayList>(); + final BitSet seen = new BitSet(); + final FST.BytesReader reader = fst.getBytesReader(); + final FST.Arc startArc = fst.getFirstArc(new FST.Arc()); + queue.add(startArc); + while (!queue.isEmpty()) { + final FST.Arc arc = queue.remove(0); + final long node = arc.target; + //System.out.println(arc); + if (FST.targetHasArcs(arc) && !seen.get((int) node)) { + seen.set((int) node); + fst.readFirstRealTargetArc(node, arc, reader); + while (true) { + queue.add(new FST.Arc().copyFrom(arc)); + if (arc.isLast()) { + break; + } else { + fst.readNextRealArc(arc, reader); + } + } + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTOrdTermsWriter.java 2013-09-02 00:34:44.698420185 +0800 @@ -0,0 +1,275 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.fst.Builder; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.PositiveIntOutputs; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.PostingsWriterBase; +import org.apache.lucene.codecs.PostingsConsumer; +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.TermsConsumer; +import org.apache.lucene.codecs.TermStats; +import org.apache.lucene.codecs.CodecUtil; + +/** + * FST based term dict, the FST maps each term and its ord. + * + * @lucene.experimental + */ + +public class TempFSTOrdTermsWriter extends FieldsConsumer { + static final String TERMS_INDEX_EXTENSION = "tix"; + static final String TERMS_BLOCK_EXTENSION = "tbk"; + static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT"; + public static final int TERMS_VERSION_START = 0; + public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START; + public static final int SKIP_INTERVAL = 8; + + final PostingsWriterBase postingsWriter; + final FieldInfos fieldInfos; + final List fields = new ArrayList(); + IndexOutput blockOut = null; + IndexOutput indexOut = null; + + public TempFSTOrdTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION); + final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_BLOCK_EXTENSION); + + this.postingsWriter = postingsWriter; + this.fieldInfos = state.fieldInfos; + + boolean success = false; + try { + this.indexOut = state.directory.createOutput(termsIndexFileName, state.context); + this.blockOut = state.directory.createOutput(termsBlockFileName, state.context); + writeHeader(indexOut); + writeHeader(blockOut); + this.postingsWriter.init(blockOut); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(indexOut, blockOut); + } + } + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + return new TermsWriter(field); + } + + @Override + public void close() throws IOException { + IOException ioe = null; + try { + final long indexDirStart = indexOut.getFilePointer(); + final long blockDirStart = blockOut.getFilePointer(); + + // write field summary + blockOut.writeVInt(fields.size()); + for (FieldMetaData field : fields) { + blockOut.writeVInt(field.fieldInfo.number); + blockOut.writeVLong(field.numTerms); + if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { + blockOut.writeVLong(field.sumTotalTermFreq); + } + blockOut.writeVLong(field.sumDocFreq); + blockOut.writeVInt(field.docCount); + blockOut.writeVInt(field.longsSize); + blockOut.writeVLong(field.statsOut.getFilePointer()); + blockOut.writeVLong(field.metaLongsOut.getFilePointer()); + blockOut.writeVLong(field.metaBytesOut.getFilePointer()); + + field.skipOut.writeTo(blockOut); + field.statsOut.writeTo(blockOut); + field.metaLongsOut.writeTo(blockOut); + field.metaBytesOut.writeTo(blockOut); + field.dict.save(indexOut); + } + writeTrailer(indexOut, indexDirStart); + writeTrailer(blockOut, blockDirStart); + } catch (IOException ioe2) { + ioe = ioe2; + } finally { + IOUtils.closeWhileHandlingException(ioe, blockOut, indexOut, postingsWriter); + } + } + + private void writeHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT); + } + private void writeTrailer(IndexOutput out, long dirStart) throws IOException { + out.writeLong(dirStart); + } + + private static class FieldMetaData { + public FieldInfo fieldInfo; + public long numTerms; + public long sumTotalTermFreq; + public long sumDocFreq; + public int docCount; + public int longsSize; + public FST dict; + + // TODO: block encode each part + + // vint encode next skip point (fully decoded when reading) + public RAMOutputStream skipOut; + // vint encode df, (ttf-df) + public RAMOutputStream statsOut; + // vint encode monotonic long[] and length for corresponding byte[] + public RAMOutputStream metaLongsOut; + // generic byte[] + public RAMOutputStream metaBytesOut; + } + + final class TermsWriter extends TermsConsumer { + private final Builder builder; + private final PositiveIntOutputs outputs; + private final FieldInfo fieldInfo; + private final int longsSize; + private long numTerms; + + private final IntsRef scratchTerm = new IntsRef(); + private final RAMOutputStream statsOut = new RAMOutputStream(); + private final RAMOutputStream metaLongsOut = new RAMOutputStream(); + private final RAMOutputStream metaBytesOut = new RAMOutputStream(); + + private final RAMOutputStream skipOut = new RAMOutputStream(); + private long lastBlockStatsFP; + private long lastBlockMetaLongsFP; + private long lastBlockMetaBytesFP; + private long[] lastBlockLongs; + + private long[] lastLongs; + private long lastMetaBytesFP; + + TermsWriter(FieldInfo fieldInfo) { + this.numTerms = 0; + this.fieldInfo = fieldInfo; + this.longsSize = postingsWriter.setField(fieldInfo); + this.outputs = PositiveIntOutputs.getSingleton(); + this.builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + + this.lastBlockStatsFP = 0; + this.lastBlockMetaLongsFP = 0; + this.lastBlockMetaBytesFP = 0; + this.lastBlockLongs = new long[longsSize]; + + this.lastLongs = new long[longsSize]; + this.lastMetaBytesFP = 0; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public PostingsConsumer startTerm(BytesRef text) throws IOException { + postingsWriter.startTerm(); + return postingsWriter; + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + if (numTerms > 0 && numTerms % SKIP_INTERVAL == 0) { + bufferSkip(); + } + // write term meta data into fst + final long longs[] = new long[longsSize]; + final long delta = stats.totalTermFreq - stats.docFreq; + if (stats.totalTermFreq > 0) { + if (delta == 0) { + statsOut.writeVInt(stats.docFreq<<1|1); + } else { + statsOut.writeVInt(stats.docFreq<<1|0); + statsOut.writeVLong(stats.totalTermFreq-stats.docFreq); + } + } else { + statsOut.writeVInt(stats.docFreq); + } + BlockTermState state = postingsWriter.newTermState(); + state.docFreq = stats.docFreq; + state.totalTermFreq = stats.totalTermFreq; + postingsWriter.finishTerm(state); + postingsWriter.encodeTerm(longs, metaBytesOut, fieldInfo, state, true); + for (int i = 0; i < longsSize; i++) { + metaLongsOut.writeVLong(longs[i] - lastLongs[i]); + lastLongs[i] = longs[i]; + } + metaLongsOut.writeVLong(metaBytesOut.getFilePointer() - lastMetaBytesFP); + + builder.add(Util.toIntsRef(text, scratchTerm), numTerms); + numTerms++; + + lastMetaBytesFP = metaBytesOut.getFilePointer(); + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { + if (numTerms > 0) { + final FieldMetaData metadata = new FieldMetaData(); + metadata.fieldInfo = fieldInfo; + metadata.numTerms = numTerms; + metadata.sumTotalTermFreq = sumTotalTermFreq; + metadata.sumDocFreq = sumDocFreq; + metadata.docCount = docCount; + metadata.longsSize = longsSize; + metadata.skipOut = skipOut; + metadata.statsOut = statsOut; + metadata.metaLongsOut = metaLongsOut; + metadata.metaBytesOut = metaBytesOut; + metadata.dict = builder.finish(); + fields.add(metadata); + } + } + + private void bufferSkip() throws IOException { + skipOut.writeVLong(statsOut.getFilePointer() - lastBlockStatsFP); + skipOut.writeVLong(metaLongsOut.getFilePointer() - lastBlockMetaLongsFP); + skipOut.writeVLong(metaBytesOut.getFilePointer() - lastBlockMetaBytesFP); + for (int i = 0; i < longsSize; i++) { + skipOut.writeVLong(lastLongs[i] - lastBlockLongs[i]); + } + lastBlockStatsFP = statsOut.getFilePointer(); + lastBlockMetaLongsFP = metaLongsOut.getFilePointer(); + lastBlockMetaBytesFP = metaBytesOut.getFilePointer(); + System.arraycopy(lastLongs, 0, lastBlockLongs, 0, longsSize); + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java 2013-09-02 00:34:44.698420185 +0800 @@ -0,0 +1,94 @@ +package org.apache.lucene.codecs.temp; + + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.PostingsWriterBase; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; + +/** + * FST-based term dict, using metadata as FST output. + * + * The FST directly holds the mapping between <term, metadata>. + * + * Term metadata consists of three parts: + * 1. term statistics: docFreq, totalTermFreq; + * 2. monotonic long[], e.g. the pointer to the postings list for that term; + * 3. generic byte[], e.g. other information need by postings reader. + * + * + * + * @lucene.experimental + */ + +public final class TempFSTPostingsFormat extends PostingsFormat { + public TempFSTPostingsFormat() { + super("TempFST"); + } + + @Override + public String toString() { + return getName(); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state); + + boolean success = false; + try { + FieldsConsumer ret = new TempFSTTermsWriter(state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsWriter); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, + state.fieldInfos, + state.segmentInfo, + state.context, + state.segmentSuffix); + boolean success = false; + try { + FieldsProducer ret = new TempFSTTermsReader(state, postingsReader); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsReader); + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPulsing41PostingsFormat.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPulsing41PostingsFormat.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPulsing41PostingsFormat.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTPulsing41PostingsFormat.java 2013-09-01 10:08:11.755684506 +0800 @@ -0,0 +1,92 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsBaseFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.PostingsWriterBase; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsBaseFormat; +import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; +import org.apache.lucene.codecs.pulsing.PulsingPostingsWriter; +import org.apache.lucene.codecs.pulsing.PulsingPostingsReader; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; + +/** TempFST + Pulsing41, test only, since + * FST does no delta encoding here! + * @lucene.experimental */ + +public class TempFSTPulsing41PostingsFormat extends PostingsFormat { + private final PostingsBaseFormat wrappedPostingsBaseFormat; + private final int freqCutoff; + + public TempFSTPulsing41PostingsFormat() { + this(1); + } + + public TempFSTPulsing41PostingsFormat(int freqCutoff) { + super("TempFSTPulsing41"); + this.wrappedPostingsBaseFormat = new Lucene41PostingsBaseFormat(); + this.freqCutoff = freqCutoff; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = null; + PostingsWriterBase pulsingWriter = null; + + boolean success = false; + try { + docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + pulsingWriter = new PulsingPostingsWriter(state, freqCutoff, docsWriter); + FieldsConsumer ret = new TempFSTTermsWriter(state, pulsingWriter); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(docsWriter, pulsingWriter); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase docsReader = null; + PostingsReaderBase pulsingReader = null; + boolean success = false; + try { + docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + pulsingReader = new PulsingPostingsReader(state, docsReader); + FieldsProducer ret = new TempFSTTermsReader(state, pulsingReader); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(docsReader, pulsingReader); + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java 2013-09-02 00:34:44.698420185 +0800 @@ -0,0 +1,732 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.File; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeMap; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.fst.BytesRefFSTEnum; +import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsReaderBase; +import org.apache.lucene.codecs.CodecUtil; + +/** + * FST-based terms dictionary reader. + * + * The FST directly maps each term and its metadata, + * it is memeory resident. + * + * @lucene.experimental + */ + +public class TempFSTTermsReader extends FieldsProducer { + final TreeMap fields = new TreeMap(); + final PostingsReaderBase postingsReader; + final IndexInput in; + //static boolean TEST = false; + + public TempFSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempFSTTermsWriter.TERMS_EXTENSION); + + this.postingsReader = postingsReader; + this.in = state.directory.openInput(termsFileName, state.context); + + boolean success = false; + try { + readHeader(in); + this.postingsReader.init(in); + seekDir(in); + + final FieldInfos fieldInfos = state.fieldInfos; + final int numFields = in.readVInt(); + for (int i = 0; i < numFields; i++) { + int fieldNumber = in.readVInt(); + FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); + long numTerms = in.readVLong(); + long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); + long sumDocFreq = in.readVLong(); + int docCount = in.readVInt(); + int longsSize = in.readVInt(); + TermsReader current = new TermsReader(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize); + TermsReader previous = fields.put(fieldInfo.name, current); + checkFieldSummary(state.segmentInfo, current, previous); + } + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(in); + } + } + } + + private int readHeader(IndexInput in) throws IOException { + return CodecUtil.checkHeader(in, TempFSTTermsWriter.TERMS_CODEC_NAME, + TempFSTTermsWriter.TERMS_VERSION_START, + TempFSTTermsWriter.TERMS_VERSION_CURRENT); + } + private void seekDir(IndexInput in) throws IOException { + in.seek(in.length() - 8); + in.seek(in.readLong()); + } + private void checkFieldSummary(SegmentInfo info, TermsReader field, TermsReader previous) throws IOException { + // #docs with field must be <= #docs + if (field.docCount < 0 || field.docCount > info.getDocCount()) { + throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")"); + } + // #postings must be >= #docs with field + if (field.sumDocFreq < field.docCount) { + throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + in + ")"); + } + // #positions must be >= #postings + if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) { + throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + in + ")"); + } + if (previous != null) { + throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + in + ")"); + } + } + + @Override + public Iterator iterator() { + return Collections.unmodifiableSet(fields.keySet()).iterator(); + } + + @Override + public Terms terms(String field) throws IOException { + assert field != null; + return fields.get(field); + } + + @Override + public int size() { + return fields.size(); + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(in, postingsReader); + } finally { + fields.clear(); + } + } + + final class TermsReader extends Terms { + final FieldInfo fieldInfo; + final long numTerms; + final long sumTotalTermFreq; + final long sumDocFreq; + final int docCount; + final int longsSize; + final FST dict; + + TermsReader(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) throws IOException { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.sumTotalTermFreq = sumTotalTermFreq; + this.sumDocFreq = sumDocFreq; + this.docCount = docCount; + this.longsSize = longsSize; + this.dict = new FST(in, new TempTermOutputs(fieldInfo, longsSize)); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public boolean hasOffsets() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + } + + @Override + public boolean hasPositions() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + } + + @Override + public boolean hasPayloads() { + return fieldInfo.hasPayloads(); + } + + @Override + public long size() { + return numTerms; + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public TermsEnum iterator(TermsEnum reuse) throws IOException { + return new SegmentTermsEnum(); + } + + @Override + public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + return new IntersectTermsEnum(compiled, startTerm); + } + + // Only wraps common operations for PBF interact + abstract class BaseTermsEnum extends TermsEnum { + /* Current term, null when enum ends or unpositioned */ + BytesRef term; + + /* Current term stats + decoded metadata (customized by PBF) */ + final BlockTermState state; + + /* Current term stats + undecoded metadata (long[] & byte[]) */ + TempTermOutputs.TempMetaData meta; + ByteArrayDataInput bytesReader; + + /** Decodes metadata into customized term state */ + abstract void decodeMetaData() throws IOException; + + BaseTermsEnum() throws IOException { + this.state = postingsReader.newTermState(); + this.bytesReader = new ByteArrayDataInput(); + this.term = null; + // NOTE: metadata will only be initialized in child class + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public TermState termState() throws IOException { + decodeMetaData(); + return state.clone(); + } + + @Override + public BytesRef term() { + return term; + } + + @Override + public int docFreq() throws IOException { + return state.docFreq; + } + + @Override + public long totalTermFreq() throws IOException { + return state.totalTermFreq; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + decodeMetaData(); + return postingsReader.docs(fieldInfo, state, liveDocs, reuse, flags); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + if (!hasPositions()) { + return null; + } + decodeMetaData(); + return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); + } + + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + } + + + // Iterates through all terms in this field + private final class SegmentTermsEnum extends BaseTermsEnum { + final BytesRefFSTEnum fstEnum; + + /* True when current term's metadata is decoded */ + boolean decoded; + + /* True when current enum is 'positioned' by seekExact(TermState) */ + boolean seekPending; + + SegmentTermsEnum() throws IOException { + super(); + this.fstEnum = new BytesRefFSTEnum(dict); + this.decoded = false; + this.seekPending = false; + this.meta = null; + } + + // Let PBF decode metadata from long[] and byte[] + @Override + void decodeMetaData() throws IOException { + if (!decoded && !seekPending) { + if (meta.bytes != null) { + bytesReader.reset(meta.bytes, 0, meta.bytes.length); + } + postingsReader.decodeTerm(meta.longs, bytesReader, fieldInfo, state, true); + decoded = true; + } + } + + // Update current enum according to FSTEnum + void updateEnum(final InputOutput pair) { + if (pair == null) { + term = null; + } else { + term = pair.input; + meta = pair.output; + state.docFreq = meta.docFreq; + state.totalTermFreq = meta.totalTermFreq; + } + decoded = false; + seekPending = false; + } + + @Override + public BytesRef next() throws IOException { + if (seekPending) { // previously positioned, but termOutputs not fetched + seekPending = false; + SeekStatus status = seekCeil(term); + assert status == SeekStatus.FOUND; // must positioned on valid term + } + updateEnum(fstEnum.next()); + return term; + } + + @Override + public boolean seekExact(BytesRef target) throws IOException { + updateEnum(fstEnum.seekExact(target)); + return term != null; + } + + @Override + public SeekStatus seekCeil(BytesRef target) throws IOException { + updateEnum(fstEnum.seekCeil(target)); + if (term == null) { + return SeekStatus.END; + } else { + return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND; + } + } + + @Override + public void seekExact(BytesRef target, TermState otherState) { + if (!target.equals(term)) { + state.copyFrom(otherState); + term = BytesRef.deepCopyOf(target); + seekPending = true; + } + } + } + + // Iterates intersect result with automaton (cannot seek!) + private final class IntersectTermsEnum extends BaseTermsEnum { + /* True when current term's metadata is decoded */ + boolean decoded; + + /* True when there is pending term when calling next() */ + boolean pending; + + /* stack to record how current term is constructed, + * used to accumulate metadata or rewind term: + * level == term.length + 1, + * == 0 when term is null */ + Frame[] stack; + int level; + + /* to which level the metadata is accumulated + * so that we can accumulate metadata lazily */ + int metaUpto; + + /* term dict fst */ + final FST fst; + final FST.BytesReader fstReader; + final Outputs fstOutputs; + + /* query automaton to intersect with */ + final ByteRunAutomaton fsa; + + private final class Frame { + /* fst stats */ + FST.Arc fstArc; + + /* automaton stats */ + int fsaState; + + Frame() { + this.fstArc = new FST.Arc(); + this.fsaState = -1; + } + + public String toString() { + return "arc=" + fstArc + " state=" + fsaState; + } + } + + IntersectTermsEnum(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + super(); + //if (TEST) System.out.println("Enum init, startTerm=" + startTerm); + this.fst = dict; + this.fstReader = fst.getBytesReader(); + this.fstOutputs = dict.outputs; + this.fsa = compiled.runAutomaton; + this.level = -1; + this.stack = new Frame[16]; + for (int i = 0 ; i < stack.length; i++) { + this.stack[i] = new Frame(); + } + + Frame frame; + frame = loadVirtualFrame(newFrame()); + this.level++; + frame = loadFirstFrame(newFrame()); + pushFrame(frame); + + this.meta = null; + this.metaUpto = 1; + this.decoded = false; + this.pending = false; + + if (startTerm == null) { + pending = isAccept(topFrame()); + } else { + doSeekCeil(startTerm); + pending = !startTerm.equals(term) && isValid(topFrame()) && isAccept(topFrame()); + } + } + + @Override + void decodeMetaData() throws IOException { + assert term != null; + if (!decoded) { + if (meta.bytes != null) { + bytesReader.reset(meta.bytes, 0, meta.bytes.length); + } + postingsReader.decodeTerm(meta.longs, bytesReader, fieldInfo, state, true); + decoded = true; + } + } + + /** Lazily accumulate meta data, when we got a accepted term */ + void loadMetaData() throws IOException { + FST.Arc last, next; + last = stack[metaUpto].fstArc; + while (metaUpto != level) { + metaUpto++; + next = stack[metaUpto].fstArc; + next.output = fstOutputs.add(next.output, last.output); + last = next; + } + if (last.isFinal()) { + meta = fstOutputs.add(last.output, last.nextFinalOutput); + } else { + meta = last.output; + } + state.docFreq = meta.docFreq; + state.totalTermFreq = meta.totalTermFreq; + } + + @Override + public SeekStatus seekCeil(BytesRef target) throws IOException { + decoded = false; + term = doSeekCeil(target); + loadMetaData(); + if (term == null) { + return SeekStatus.END; + } else { + return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND; + } + } + + @Override + public BytesRef next() throws IOException { + //if (TEST) System.out.println("Enum next()"); + if (pending) { + pending = false; + loadMetaData(); + return term; + } + decoded = false; + DFS: + while (level > 0) { + Frame frame = newFrame(); + if (loadExpandFrame(topFrame(), frame) != null) { // has valid target + pushFrame(frame); + if (isAccept(frame)) { // gotcha + break; + } + continue; // check next target + } + frame = popFrame(); + while(level > 0) { + if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling + pushFrame(frame); + if (isAccept(frame)) { // gotcha + break DFS; + } + continue DFS; // check next target + } + frame = popFrame(); + } + return null; + } + loadMetaData(); + return term; + } + + private BytesRef doSeekCeil(BytesRef target) throws IOException { + //if (TEST) System.out.println("Enum doSeekCeil()"); + Frame frame= null; + int label, upto = 0, limit = target.length; + while (upto < limit) { // to target prefix, or ceil label (rewind prefix) + frame = newFrame(); + label = target.bytes[upto] & 0xff; + frame = loadCeilFrame(label, topFrame(), frame); + if (frame == null || frame.fstArc.label != label) { + break; + } + assert isValid(frame); // target must be fetched from automaton + pushFrame(frame); + upto++; + } + if (upto == limit) { // got target + return term; + } + if (frame != null) { // got larger term('s prefix) + pushFrame(frame); + return isAccept(frame) ? term : next(); + } + while (level > 0) { // got target's prefix, advance to larger term + frame = popFrame(); + while (level > 0 && !canRewind(frame)) { + frame = popFrame(); + } + if (loadNextFrame(topFrame(), frame) != null) { + pushFrame(frame); + return isAccept(frame) ? term : next(); + } + } + return null; + } + + /** Virtual frame, never pop */ + Frame loadVirtualFrame(Frame frame) throws IOException { + frame.fstArc.output = fstOutputs.getNoOutput(); + frame.fstArc.nextFinalOutput = fstOutputs.getNoOutput(); + frame.fsaState = -1; + return frame; + } + + /** Load frame for start arc(node) on fst */ + Frame loadFirstFrame(Frame frame) throws IOException { + frame.fstArc = fst.getFirstArc(frame.fstArc); + frame.fsaState = fsa.getInitialState(); + return frame; + } + + /** Load frame for target arc(node) on fst */ + Frame loadExpandFrame(Frame top, Frame frame) throws IOException { + if (!canGrow(top)) { + return null; + } + frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target, frame.fstArc, fstReader); + frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); + //if (TEST) System.out.println(" loadExpand frame="+frame); + if (frame.fsaState == -1) { + return loadNextFrame(top, frame); + } + return frame; + } + + /** Load frame for sibling arc(node) on fst */ + Frame loadNextFrame(Frame top, Frame frame) throws IOException { + if (!canRewind(frame)) { + return null; + } + while (!frame.fstArc.isLast()) { + frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader); + frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); + if (frame.fsaState != -1) { + break; + } + } + //if (TEST) System.out.println(" loadNext frame="+frame); + if (frame.fsaState == -1) { + return null; + } + return frame; + } + + /** Load frame for target arc(node) on fst, so that + * arc.label >= label and !fsa.reject(arc.label) */ + Frame loadCeilFrame(int label, Frame top, Frame frame) throws IOException { + FST.Arc arc = frame.fstArc; + arc = Util.readCeilArc(label, fst, top.fstArc, arc, fstReader); + if (arc == null) { + return null; + } + frame.fsaState = fsa.step(top.fsaState, arc.label); + //if (TEST) System.out.println(" loadCeil frame="+frame); + if (frame.fsaState == -1) { + return loadNextFrame(top, frame); + } + return frame; + } + + boolean isAccept(Frame frame) { // reach a term both fst&fsa accepts + return fsa.isAccept(frame.fsaState) && frame.fstArc.isFinal(); + } + boolean isValid(Frame frame) { // reach a prefix both fst&fsa won't reject + return /*frame != null &&*/ frame.fsaState != -1; + } + boolean canGrow(Frame frame) { // can walk forward on both fst&fsa + return frame.fsaState != -1 && FST.targetHasArcs(frame.fstArc); + } + boolean canRewind(Frame frame) { // can jump to sibling + return !frame.fstArc.isLast(); + } + + void pushFrame(Frame frame) { + term = grow(frame.fstArc.label); + level++; + //if (TEST) System.out.println(" term=" + term + " level=" + level); + } + + Frame popFrame() { + term = shrink(); + level--; + metaUpto = metaUpto > level ? level : metaUpto; + //if (TEST) System.out.println(" term=" + term + " level=" + level); + return stack[level+1]; + } + + Frame newFrame() { + if (level+1 == stack.length) { + final Frame[] temp = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + System.arraycopy(stack, 0, temp, 0, stack.length); + for (int i = stack.length; i < temp.length; i++) { + temp[i] = new Frame(); + } + stack = temp; + } + return stack[level+1]; + } + + Frame topFrame() { + return stack[level]; + } + + BytesRef grow(int label) { + if (term == null) { + term = new BytesRef(new byte[16], 0, 0); + } else { + if (term.length == term.bytes.length) { + term.grow(term.length+1); + } + term.bytes[term.length++] = (byte)label; + } + return term; + } + + BytesRef shrink() { + if (term.length == 0) { + term = null; + } else { + term.length--; + } + return term; + } + } + } + + static void walk(FST fst) throws IOException { + final ArrayList> queue = new ArrayList>(); + final BitSet seen = new BitSet(); + final FST.BytesReader reader = fst.getBytesReader(); + final FST.Arc startArc = fst.getFirstArc(new FST.Arc()); + queue.add(startArc); + while (!queue.isEmpty()) { + final FST.Arc arc = queue.remove(0); + final long node = arc.target; + //System.out.println(arc); + if (FST.targetHasArcs(arc) && !seen.get((int) node)) { + seen.set((int) node); + fst.readFirstRealTargetArc(node, arc, reader); + while (true) { + queue.add(new FST.Arc().copyFrom(arc)); + if (arc.isLast()) { + break; + } else { + fst.readNextRealArc(arc, reader); + } + } + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java 2013-09-02 00:34:44.698420185 +0800 @@ -0,0 +1,201 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.fst.Builder; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.PostingsWriterBase; +import org.apache.lucene.codecs.PostingsConsumer; +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.TermsConsumer; +import org.apache.lucene.codecs.TermStats; +import org.apache.lucene.codecs.CodecUtil; + +/** + * FST based term dict, the FST maps each term and its metadata. + * + * @lucene.experimental + */ + +public class TempFSTTermsWriter extends FieldsConsumer { + static final String TERMS_EXTENSION = "tmp"; + static final String TERMS_CODEC_NAME = "FST_TERMS_DICT"; + public static final int TERMS_VERSION_START = 0; + public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START; + + final PostingsWriterBase postingsWriter; + final FieldInfos fieldInfos; + final IndexOutput out; + final List fields = new ArrayList(); + + public TempFSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION); + + this.postingsWriter = postingsWriter; + this.fieldInfos = state.fieldInfos; + this.out = state.directory.createOutput(termsFileName, state.context); + + boolean success = false; + try { + writeHeader(out); + this.postingsWriter.init(out); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + private void writeHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT); + } + private void writeTrailer(IndexOutput out, long dirStart) throws IOException { + out.writeLong(dirStart); + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + return new TermsWriter(field); + } + + @Override + public void close() throws IOException { + IOException ioe = null; + try { + // write field summary + final long dirStart = out.getFilePointer(); + + out.writeVInt(fields.size()); + for (FieldMetaData field : fields) { + out.writeVInt(field.fieldInfo.number); + out.writeVLong(field.numTerms); + if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { + out.writeVLong(field.sumTotalTermFreq); + } + out.writeVLong(field.sumDocFreq); + out.writeVInt(field.docCount); + out.writeVInt(field.longsSize); + field.dict.save(out); + } + writeTrailer(out, dirStart); + } catch (IOException ioe2) { + ioe = ioe2; + } finally { + IOUtils.closeWhileHandlingException(ioe, out, postingsWriter); + } + } + + private static class FieldMetaData { + public final FieldInfo fieldInfo; + public final long numTerms; + public final long sumTotalTermFreq; + public final long sumDocFreq; + public final int docCount; + public final int longsSize; + public final FST dict; + + public FieldMetaData(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST fst) { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.sumTotalTermFreq = sumTotalTermFreq; + this.sumDocFreq = sumDocFreq; + this.docCount = docCount; + this.longsSize = longsSize; + this.dict = fst; + } + } + + final class TermsWriter extends TermsConsumer { + private final Builder builder; + private final TempTermOutputs outputs; + private final FieldInfo fieldInfo; + private final int longsSize; + private long numTerms; + + private final IntsRef scratchTerm = new IntsRef(); + private final RAMOutputStream statsWriter = new RAMOutputStream(); + private final RAMOutputStream metaWriter = new RAMOutputStream(); + + TermsWriter(FieldInfo fieldInfo) { + this.numTerms = 0; + this.fieldInfo = fieldInfo; + this.longsSize = postingsWriter.setField(fieldInfo); + this.outputs = new TempTermOutputs(fieldInfo, longsSize); + this.builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public PostingsConsumer startTerm(BytesRef text) throws IOException { + postingsWriter.startTerm(); + return postingsWriter; + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + // write term meta data into fst + final BlockTermState state = postingsWriter.newTermState(); + final TempTermOutputs.TempMetaData meta = new TempTermOutputs.TempMetaData(); + meta.longs = new long[longsSize]; + meta.bytes = null; + meta.docFreq = state.docFreq = stats.docFreq; + meta.totalTermFreq = state.totalTermFreq = stats.totalTermFreq; + postingsWriter.finishTerm(state); + postingsWriter.encodeTerm(meta.longs, metaWriter, fieldInfo, state, true); + final int bytesSize = (int)metaWriter.getFilePointer(); + if (bytesSize > 0) { + meta.bytes = new byte[bytesSize]; + metaWriter.writeTo(meta.bytes, 0); + metaWriter.reset(); + } + builder.add(Util.toIntsRef(text, scratchTerm), meta); + numTerms++; + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { + // save FST dict + if (numTerms > 0) { + final FST fst = builder.finish(); + fields.add(new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, fst)); + } + } + } +} diff -ruN -x .svn -x build trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java --- trunk/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java 1970-01-01 08:00:00.000000000 +0800 +++ branch3069/lucene/codecs/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java 2013-09-03 19:43:02.153301716 +0800 @@ -0,0 +1,331 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.LongsRef; + +/** + * An FST {@link Outputs} implementation for + * {@link TempFSTPostingsFormat}. + * + * @lucene.experimental + */ + +// NOTE: outputs should be per-field, since +// longsSize is fixed for each field +public class TempTermOutputs extends Outputs { + private final static TempMetaData NO_OUTPUT = new TempMetaData(); + //private static boolean TEST = false; + private final boolean hasPos; + private final int longsSize; + + /** + * Represents the metadata for one term. + * On an FST, only long[] part is 'shared' and pushed towards root. + * byte[] and term stats will be kept on deeper arcs. + */ + public static class TempMetaData { + long[] longs; + byte[] bytes; + int docFreq; + long totalTermFreq; + TempMetaData() { + this.longs = null; + this.bytes = null; + this.docFreq = 0; + this.totalTermFreq = -1; + } + TempMetaData(long[] longs, byte[] bytes, int docFreq, long totalTermFreq) { + this.longs = longs; + this.bytes = bytes; + this.docFreq = docFreq; + this.totalTermFreq = totalTermFreq; + } + + // NOTE: actually, FST nodes are seldom + // identical when outputs on their arcs + // aren't NO_OUTPUTs. + @Override + public int hashCode() { + int hash = 0; + if (longs != null) { + final int end = longs.length; + for (int i = 0; i < end; i++) { + hash -= longs[i]; + } + } + if (bytes != null) { + hash = -hash; + final int end = bytes.length; + for (int i = 0; i < end; i++) { + hash += bytes[i]; + } + } + hash += docFreq + totalTermFreq; + return hash; + } + + @Override + public boolean equals(Object other_) { + if (other_ == this) { + return true; + } else if (!(other_ instanceof TempTermOutputs.TempMetaData)) { + return false; + } + TempMetaData other = (TempMetaData) other_; + return statsEqual(this, other) && + longsEqual(this, other) && + bytesEqual(this, other); + + } + } + + protected TempTermOutputs(FieldInfo fieldInfo, int longsSize) { + this.hasPos = (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY); + this.longsSize = longsSize; + } + + @Override + // + // The return value will be the smaller one, when these two are + // 'comparable', i.e. + // 1. every value in t1 is not larger than in t2, or + // 2. every value in t1 is not smaller than t2. + // + public TempMetaData common(TempMetaData t1, TempMetaData t2) { + //if (TEST) System.out.print("common("+t1+", "+t2+") = "); + if (t1 == NO_OUTPUT || t2 == NO_OUTPUT) { + //if (TEST) System.out.println("ret:"+NO_OUTPUT); + return NO_OUTPUT; + } + assert t1.longs.length == t2.longs.length; + + long[] min = t1.longs, max = t2.longs; + int pos = 0; + TempMetaData ret; + + while (pos < longsSize && min[pos] == max[pos]) { + pos++; + } + if (pos < longsSize) { // unequal long[] + if (min[pos] > max[pos]) { + min = t2.longs; + max = t1.longs; + } + // check whether strictly smaller + while (pos < longsSize && min[pos] <= max[pos]) { + pos++; + } + if (pos < longsSize || allZero(min)) { // not comparable or all-zero + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(min, null, 0, -1); + } + } else { // equal long[] + if (statsEqual(t1, t2) && bytesEqual(t1, t2)) { + ret = t1; + } else if (allZero(min)) { + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(min, null, 0, -1); + } + } + //if (TEST) System.out.println("ret:"+ret); + return ret; + } + + @Override + public TempMetaData subtract(TempMetaData t1, TempMetaData t2) { + //if (TEST) System.out.print("subtract("+t1+", "+t2+") = "); + if (t2 == NO_OUTPUT) { + //if (TEST) System.out.println("ret:"+t1); + return t1; + } + assert t1.longs.length == t2.longs.length; + + int pos = 0; + long diff = 0; + long[] share = new long[longsSize]; + + while (pos < longsSize) { + share[pos] = t1.longs[pos] - t2.longs[pos]; + diff += share[pos]; + pos++; + } + + TempMetaData ret; + if (diff == 0 && statsEqual(t1, t2) && bytesEqual(t1, t2)) { + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(share, t1.bytes, t1.docFreq, t1.totalTermFreq); + } + //if (TEST) System.out.println("ret:"+ret); + return ret; + } + + // TODO: if we refactor a 'addSelf(TempMetaDat other)', + // we can gain about 5~7% for fuzzy queries, however this also + // means we are putting too much stress on FST Outputs decoding? + @Override + public TempMetaData add(TempMetaData t1, TempMetaData t2) { + //if (TEST) System.out.print("add("+t1+", "+t2+") = "); + if (t1 == NO_OUTPUT) { + //if (TEST) System.out.println("ret:"+t2); + return t2; + } else if (t2 == NO_OUTPUT) { + //if (TEST) System.out.println("ret:"+t1); + return t1; + } + assert t1.longs.length == t2.longs.length; + + int pos = 0; + long[] accum = new long[longsSize]; + + while (pos < longsSize) { + accum[pos] = t1.longs[pos] + t2.longs[pos]; + pos++; + } + + TempMetaData ret; + if (t2.bytes != null || t2.docFreq > 0) { + ret = new TempMetaData(accum, t2.bytes, t2.docFreq, t2.totalTermFreq); + } else { + ret = new TempMetaData(accum, t1.bytes, t1.docFreq, t1.totalTermFreq); + } + //if (TEST) System.out.println("ret:"+ret); + return ret; + } + + @Override + public void write(TempMetaData data, DataOutput out) throws IOException { + int bit0 = allZero(data.longs) ? 0 : 1; + int bit1 = ((data.bytes == null || data.bytes.length == 0) ? 0 : 1) << 1; + int bit2 = ((data.docFreq == 0) ? 0 : 1) << 2; + int bits = bit0 | bit1 | bit2; + if (bit1 > 0) { // determine extra length + if (data.bytes.length < 32) { + bits |= (data.bytes.length << 3); + out.writeByte((byte)bits); + } else { + out.writeByte((byte)bits); + out.writeVInt(data.bytes.length); + } + } else { + out.writeByte((byte)bits); + } + if (bit0 > 0) { // not all-zero case + for (int pos = 0; pos < longsSize; pos++) { + out.writeVLong(data.longs[pos]); + } + } + if (bit1 > 0) { // bytes exists + out.writeBytes(data.bytes, 0, data.bytes.length); + } + if (bit2 > 0) { // stats exist + if (hasPos) { + if (data.docFreq == data.totalTermFreq) { + out.writeVInt((data.docFreq << 1) | 1); + } else { + out.writeVInt((data.docFreq << 1)); + out.writeVLong(data.totalTermFreq - data.docFreq); + } + } else { + out.writeVInt(data.docFreq); + } + } + } + + @Override + public TempMetaData read(DataInput in) throws IOException { + long[] longs = new long[longsSize]; + byte[] bytes = null; + int docFreq = 0; + long totalTermFreq = -1; + int bits = in.readByte() & 0xff; + int bit0 = bits & 1; + int bit1 = bits & 2; + int bit2 = bits & 4; + int bytesSize = (bits >>> 3); + if (bit1 > 0 && bytesSize == 0) { // determine extra length + bytesSize = in.readVInt(); + } + if (bit0 > 0) { // not all-zero case + for (int pos = 0; pos < longsSize; pos++) { + longs[pos] = in.readVLong(); + } + } + if (bit1 > 0) { // bytes exists + bytes = new byte[bytesSize]; + in.readBytes(bytes, 0, bytesSize); + } + if (bit2 > 0) { // stats exist + int code = in.readVInt(); + if (hasPos) { + totalTermFreq = docFreq = code >>> 1; + if ((code & 1) == 0) { + totalTermFreq += in.readVLong(); + } + } else { + docFreq = code; + } + } + return new TempMetaData(longs, bytes, docFreq, totalTermFreq); + } + + @Override + public TempMetaData getNoOutput() { + return NO_OUTPUT; + } + + @Override + public String outputToString(TempMetaData data) { + return data.toString(); + } + + static boolean statsEqual(final TempMetaData t1, final TempMetaData t2) { + return t1.docFreq == t2.docFreq && t1.totalTermFreq == t2.totalTermFreq; + } + static boolean bytesEqual(final TempMetaData t1, final TempMetaData t2) { + if (t1.bytes == null && t2.bytes == null) { + return true; + } + return t1.bytes != null && t2.bytes != null && Arrays.equals(t1.bytes, t2.bytes); + } + static boolean longsEqual(final TempMetaData t1, final TempMetaData t2) { + if (t1.longs == null && t2.longs == null) { + return true; + } + return t1.longs != null && t2.longs != null && Arrays.equals(t1.longs, t2.longs); + } + static boolean allZero(final long[] l) { + for (int i = 0; i < l.length; i++) { + if (l[i] != 0) { + return false; + } + } + return true; + } +} diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java 2013-09-01 19:36:45.911959032 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java 2013-09-01 10:08:11.702351174 +0800 @@ -34,6 +34,7 @@ /** the term's ord in the current block */ public int termBlockOrd; /** fp into the terms dict primary file (_X.tim) that holds this term */ + // TODO: update BTR to nuke this public long blockFilePointer; /** Sole constructor. (For invocation by subclass diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java 2013-09-01 19:36:45.915292366 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java 2013-08-26 22:50:46.246824736 +0800 @@ -158,6 +158,7 @@ final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); final long sumDocFreq = in.readVLong(); final int docCount = in.readVInt(); + final int longsSize = version >= BlockTreeTermsWriter.TERMS_VERSION_META_ARRAY ? in.readVInt() : 0; if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")"); } @@ -168,7 +169,7 @@ throw new CorruptIndexException("invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq + " (resource=" + in + ")"); } final long indexStartFP = indexIn.readVLong(); - FieldReader previous = fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, indexStartFP, indexIn)); + FieldReader previous = fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, indexStartFP, longsSize, indexIn)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" + in + ")"); } @@ -448,11 +449,12 @@ final long indexStartFP; final long rootBlockFP; final BytesRef rootCode; - private final FST index; + final int longsSize; + private final FST index; //private boolean DEBUG; - FieldReader(FieldInfo fieldInfo, long numTerms, BytesRef rootCode, long sumTotalTermFreq, long sumDocFreq, int docCount, long indexStartFP, IndexInput indexIn) throws IOException { + FieldReader(FieldInfo fieldInfo, long numTerms, BytesRef rootCode, long sumTotalTermFreq, long sumDocFreq, int docCount, long indexStartFP, int longsSize, IndexInput indexIn) throws IOException { assert numTerms > 0; this.fieldInfo = fieldInfo; //DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id"); @@ -462,6 +464,7 @@ this.docCount = docCount; this.indexStartFP = indexStartFP; this.rootCode = rootCode; + this.longsSize = longsSize; // if (DEBUG) { // System.out.println("BTTR: seg=" + segment + " field=" + fieldInfo.name + " rootBlockCode=" + rootCode + " divisor=" + indexDivisor); // } @@ -612,6 +615,12 @@ FST.Arc arc; final BlockTermState termState; + + // metadata buffer, holding monotonical values + public long[] longs; + // metadata buffer, holding general values + public byte[] bytes; + ByteArrayDataInput bytesReader; // Cumulative output so far BytesRef outputPrefix; @@ -621,8 +630,9 @@ public Frame(int ord) throws IOException { this.ord = ord; - termState = postingsReader.newTermState(); - termState.totalTermFreq = -1; + this.termState = postingsReader.newTermState(); + this.termState.totalTermFreq = -1; + this.longs = new long[longsSize]; } void loadNextFloorBlock() throws IOException { @@ -720,8 +730,17 @@ termState.termBlockOrd = 0; nextEnt = 0; - - postingsReader.readTermsBlock(in, fieldInfo, termState); + + // metadata + numBytes = in.readVInt(); + if (bytes == null) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + bytesReader = new ByteArrayDataInput(); + } else if (bytes.length < numBytes) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + } + in.readBytes(bytes, 0, numBytes); + bytesReader.reset(bytes, 0, numBytes); if (!isLastInFloor) { // Sub-blocks of a single floor block are always @@ -774,12 +793,9 @@ // lazily catch up on metadata decode: final int limit = getTermBlockOrd(); + boolean absolute = metaDataUpto == 0; assert limit > 0; - // We must set/incr state.termCount because - // postings impl can look at this - termState.termBlockOrd = metaDataUpto; - // TODO: better API would be "jump straight to term=N"??? while (metaDataUpto < limit) { @@ -791,17 +807,24 @@ // TODO: if docFreq were bulk decoded we could // just skipN here: + + // stats termState.docFreq = statsReader.readVInt(); //if (DEBUG) System.out.println(" dF=" + state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { termState.totalTermFreq = termState.docFreq + statsReader.readVLong(); //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq); } + // metadata + for (int i = 0; i < longsSize; i++) { + longs[i] = bytesReader.readVLong(); + } + postingsReader.decodeTerm(longs, bytesReader, fieldInfo, termState, absolute); - postingsReader.nextTerm(fieldInfo, termState); metaDataUpto++; - termState.termBlockOrd++; + absolute = false; } + termState.termBlockOrd = metaDataUpto; } } @@ -1707,6 +1730,7 @@ if (arc.output != NO_OUTPUT) { output = fstOutputs.add(output, arc.output); } + // if (DEBUG) { // System.out.println(" index: follow label=" + toHex(target.bytes[target.offset + targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput); // } @@ -2290,10 +2314,17 @@ final BlockTermState state; + // metadata buffer, holding monotonical values + public long[] longs; + // metadata buffer, holding general values + public byte[] bytes; + ByteArrayDataInput bytesReader; + public Frame(int ord) throws IOException { this.ord = ord; - state = postingsReader.newTermState(); - state.totalTermFreq = -1; + this.state = postingsReader.newTermState(); + this.state.totalTermFreq = -1; + this.longs = new long[longsSize]; } public void setFloorData(ByteArrayDataInput in, BytesRef source) { @@ -2391,7 +2422,17 @@ // TODO: we could skip this if !hasTerms; but // that's rare so won't help much - postingsReader.readTermsBlock(in, fieldInfo, state); + // metadata + numBytes = in.readVInt(); + if (bytes == null) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + bytesReader = new ByteArrayDataInput(); + } else if (bytes.length < numBytes) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + } + in.readBytes(bytes, 0, numBytes); + bytesReader.reset(bytes, 0, numBytes); + // Sub-blocks of a single floor block are always // written one after another -- tail recurse: @@ -2575,12 +2616,9 @@ // lazily catch up on metadata decode: final int limit = getTermBlockOrd(); + boolean absolute = metaDataUpto == 0; assert limit > 0; - // We must set/incr state.termCount because - // postings impl can look at this - state.termBlockOrd = metaDataUpto; - // TODO: better API would be "jump straight to term=N"??? while (metaDataUpto < limit) { @@ -2592,17 +2630,24 @@ // TODO: if docFreq were bulk decoded we could // just skipN here: + + // stats state.docFreq = statsReader.readVInt(); //if (DEBUG) System.out.println(" dF=" + state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { state.totalTermFreq = state.docFreq + statsReader.readVLong(); //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq); } + // metadata + for (int i = 0; i < longsSize; i++) { + longs[i] = bytesReader.readVLong(); + } + postingsReader.decodeTerm(longs, bytesReader, fieldInfo, state, absolute); - postingsReader.nextTerm(fieldInfo, state); metaDataUpto++; - state.termBlockOrd++; + absolute = false; } + state.termBlockOrd = metaDataUpto; } // Used only by assert diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java 2013-09-01 19:36:45.921959033 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java 2013-09-03 19:43:02.246635047 +0800 @@ -104,13 +104,12 @@ * and decoding the Postings Metadata and Term Metadata sections.

* *
    - * - *
  • TermsDict (.tim) --> Header, Postings Metadata, BlockNumBlocks, + *
  • TermsDict (.tim) --> Header, Postings Header, NodeBlockNumBlocks, * FieldSummary, DirOffset
    • - *
  • Block --> SuffixBlock, StatsBlock, MetadataBlock
    • - *
  • SuffixBlock --> EntryCount, SuffixLength, ByteSuffixLength
    • - *
  • StatsBlock --> StatsLength, <DocFreq, TotalTermFreq>EntryCount
    • - *
  • MetadataBlock --> MetaLength, <Term Metadata>EntryCount
    • + *
  • NodeBlock --> (OuterNode | InnerNode)
    • + *
  • OuterNode --> EntryCount, SuffixLength, ByteSuffixLength, StatsLength, < TermStats >EntryCount, MetaLength, <Term Metadata>EntryCount
    • + *
  • InnerNode --> EntryCount, SuffixLength[,Sub?], ByteSuffixLength, StatsLength, < TermStats ? >EntryCount, MetaLength, <Term Metadata ? >EntryCount
    • + *
  • TermStats --> DocFreq, TotalTermFreq
    • *
  • FieldSummary --> NumFields, <FieldNumber, NumTerms, RootCodeLength, ByteRootCodeLength, * SumDocFreq, DocCount>NumFields
    • *
  • Header --> {@link CodecUtil#writeHeader CodecHeader}
    • @@ -136,7 +135,9 @@ *
  • DocCount is the number of documents that have at least one posting for this field.
    • *
  • PostingsMetadata and TermMetadata are plugged into by the specific postings implementation: * these contain arbitrary per-file data (such as parameters or versioning information) - * and per-term data (such as pointers to inverted files). + * and per-term data (such as pointers to inverted files).
    • + *
  • For inner nodes of the tree, every entry will steal one bit to mark whether it points + * to child nodes(sub-block). If so, the corresponding TermStats and TermMetaData are omitted
    • *
* *

Term Index

@@ -204,8 +205,11 @@ /** Append-only */ public static final int TERMS_VERSION_APPEND_ONLY = 1; + /** Meta data as array */ + public static final int TERMS_VERSION_META_ARRAY = 2; + /** Current terms format. */ - public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_APPEND_ONLY; + public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_META_ARRAY; /** Extension of terms index file */ static final String TERMS_INDEX_EXTENSION = "tip"; @@ -217,8 +221,11 @@ /** Append-only */ public static final int TERMS_INDEX_VERSION_APPEND_ONLY = 1; + /** Meta data as array */ + public static final int TERMS_INDEX_VERSION_META_ARRAY = 2; + /** Current index format. */ - public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_APPEND_ONLY; + public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_META_ARRAY; private final IndexOutput out; private final IndexOutput indexOut; @@ -237,8 +244,9 @@ public final long sumTotalTermFreq; public final long sumDocFreq; public final int docCount; + private final int longsSize; - public FieldMetaData(FieldInfo fieldInfo, BytesRef rootCode, long numTerms, long indexStartFP, long sumTotalTermFreq, long sumDocFreq, int docCount) { + public FieldMetaData(FieldInfo fieldInfo, BytesRef rootCode, long numTerms, long indexStartFP, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) { assert numTerms > 0; this.fieldInfo = fieldInfo; assert rootCode != null: "field=" + fieldInfo.name + " numTerms=" + numTerms; @@ -248,6 +256,7 @@ this.sumTotalTermFreq = sumTotalTermFreq; this.sumDocFreq = sumDocFreq; this.docCount = docCount; + this.longsSize = longsSize; } } @@ -300,7 +309,7 @@ // System.out.println("BTW.init seg=" + state.segmentName); - postingsWriter.start(out); // have consumer write its format/header + postingsWriter.init(out); // have consumer write its format/header success = true; } finally { if (!success) { @@ -354,12 +363,13 @@ private static final class PendingTerm extends PendingEntry { public final BytesRef term; - public final TermStats stats; + // stats + metadata + public final BlockTermState state; - public PendingTerm(BytesRef term, TermStats stats) { + public PendingTerm(BytesRef term, BlockTermState state) { super(true); this.term = term; - this.stats = stats; + this.state = state; } @Override @@ -480,6 +490,7 @@ class TermsWriter extends TermsConsumer { private final FieldInfo fieldInfo; + private final int longsSize; private long numTerms; long sumTotalTermFreq; long sumDocFreq; @@ -839,11 +850,16 @@ final List> subIndices; int termCount; + + long[] longs = new long[longsSize]; + boolean absolute = true; + if (isLeafBlock) { subIndices = null; for (PendingEntry ent : slice) { assert ent.isTerm; PendingTerm term = (PendingTerm) ent; + BlockTermState state = term.state; final int suffix = term.term.length - prefixLength; // if (DEBUG) { // BytesRef suffixBytes = new BytesRef(suffix); @@ -852,15 +868,25 @@ // System.out.println(" write term suffix=" + suffixBytes); // } // For leaf block we write suffix straight - bytesWriter.writeVInt(suffix); - bytesWriter.writeBytes(term.term.bytes, prefixLength, suffix); + suffixWriter.writeVInt(suffix); + suffixWriter.writeBytes(term.term.bytes, prefixLength, suffix); // Write term stats, to separate byte[] blob: - bytesWriter2.writeVInt(term.stats.docFreq); + statsWriter.writeVInt(state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { - assert term.stats.totalTermFreq >= term.stats.docFreq: term.stats.totalTermFreq + " vs " + term.stats.docFreq; - bytesWriter2.writeVLong(term.stats.totalTermFreq - term.stats.docFreq); + assert state.totalTermFreq >= state.docFreq: state.totalTermFreq + " vs " + state.docFreq; + statsWriter.writeVLong(state.totalTermFreq - state.docFreq); + } + + // Write term meta data + postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); + for (int pos = 0; pos < longsSize; pos++) { + assert longs[pos] >= 0; + metaWriter.writeVLong(longs[pos]); } + bytesWriter.writeTo(metaWriter); + bytesWriter.reset(); + absolute = false; } termCount = length; } else { @@ -869,6 +895,7 @@ for (PendingEntry ent : slice) { if (ent.isTerm) { PendingTerm term = (PendingTerm) ent; + BlockTermState state = term.state; final int suffix = term.term.length - prefixLength; // if (DEBUG) { // BytesRef suffixBytes = new BytesRef(suffix); @@ -878,16 +905,34 @@ // } // For non-leaf block we borrow 1 bit to record // if entry is term or sub-block - bytesWriter.writeVInt(suffix<<1); - bytesWriter.writeBytes(term.term.bytes, prefixLength, suffix); + suffixWriter.writeVInt(suffix<<1); + suffixWriter.writeBytes(term.term.bytes, prefixLength, suffix); // Write term stats, to separate byte[] blob: - bytesWriter2.writeVInt(term.stats.docFreq); + statsWriter.writeVInt(state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { - assert term.stats.totalTermFreq >= term.stats.docFreq; - bytesWriter2.writeVLong(term.stats.totalTermFreq - term.stats.docFreq); + assert state.totalTermFreq >= state.docFreq; + statsWriter.writeVLong(state.totalTermFreq - state.docFreq); } + // TODO: now that terms dict "sees" these longs, + // we can explore better column-stride encodings + // to encode all long[0]s for this block at + // once, all long[1]s, etc., e.g. using + // Simple64. Alternatively, we could interleave + // stats + meta ... no reason to have them + // separate anymore: + + // Write term meta data + postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); + for (int pos = 0; pos < longsSize; pos++) { + assert longs[pos] >= 0; + metaWriter.writeVLong(longs[pos]); + } + bytesWriter.writeTo(metaWriter); + bytesWriter.reset(); + absolute = false; + termCount++; } else { PendingBlock block = (PendingBlock) ent; @@ -897,8 +942,8 @@ // For non-leaf block we borrow 1 bit to record // if entry is term or sub-block - bytesWriter.writeVInt((suffix<<1)|1); - bytesWriter.writeBytes(block.prefix.bytes, prefixLength, suffix); + suffixWriter.writeVInt((suffix<<1)|1); + suffixWriter.writeBytes(block.prefix.bytes, prefixLength, suffix); assert block.fp < startFP; // if (DEBUG) { @@ -908,7 +953,7 @@ // System.out.println(" write sub-block suffix=" + toString(suffixBytes) + " subFP=" + block.fp + " subCode=" + (startFP-block.fp) + " floor=" + block.isFloor); // } - bytesWriter.writeVLong(startFP - block.fp); + suffixWriter.writeVLong(startFP - block.fp); subIndices.add(block.index); } } @@ -921,17 +966,19 @@ // search on lookup // Write suffixes byte[] blob to terms dict output: - out.writeVInt((int) (bytesWriter.getFilePointer() << 1) | (isLeafBlock ? 1:0)); - bytesWriter.writeTo(out); - bytesWriter.reset(); + out.writeVInt((int) (suffixWriter.getFilePointer() << 1) | (isLeafBlock ? 1:0)); + suffixWriter.writeTo(out); + suffixWriter.reset(); // Write term stats byte[] blob - out.writeVInt((int) bytesWriter2.getFilePointer()); - bytesWriter2.writeTo(out); - bytesWriter2.reset(); - - // Have postings writer write block - postingsWriter.flushTermsBlock(futureTermCount+termCount, termCount); + out.writeVInt((int) statsWriter.getFilePointer()); + statsWriter.writeTo(out); + statsWriter.reset(); + + // Write term meta data byte[] blob + out.writeVInt((int) metaWriter.getFilePointer()); + metaWriter.writeTo(out); + metaWriter.reset(); // Remove slice replaced by block: slice.clear(); @@ -967,7 +1014,7 @@ PackedInts.COMPACT, true, 15); - postingsWriter.setField(fieldInfo); + this.longsSize = postingsWriter.setField(fieldInfo); } @Override @@ -998,8 +1045,13 @@ //if (DEBUG) System.out.println("BTTW.finishTerm term=" + fieldInfo.name + ":" + toString(text) + " seg=" + segment + " df=" + stats.docFreq); blockBuilder.add(Util.toIntsRef(text, scratchIntsRef), noOutputs.getNoOutput()); - pending.add(new PendingTerm(BytesRef.deepCopyOf(text), stats)); - postingsWriter.finishTerm(stats); + BlockTermState state = postingsWriter.newTermState(); + state.docFreq = stats.docFreq; + state.totalTermFreq = stats.totalTermFreq; + postingsWriter.finishTerm(state); + + PendingTerm term = new PendingTerm(BytesRef.deepCopyOf(text), state); + pending.add(term); numTerms++; } @@ -1038,7 +1090,8 @@ indexStartFP, sumTotalTermFreq, sumDocFreq, - docCount)); + docCount, + longsSize)); } else { assert sumTotalTermFreq == 0 || fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY && sumTotalTermFreq == -1; assert sumDocFreq == 0; @@ -1046,8 +1099,10 @@ } } + private final RAMOutputStream suffixWriter = new RAMOutputStream(); + private final RAMOutputStream statsWriter = new RAMOutputStream(); + private final RAMOutputStream metaWriter = new RAMOutputStream(); private final RAMOutputStream bytesWriter = new RAMOutputStream(); - private final RAMOutputStream bytesWriter2 = new RAMOutputStream(); } @Override @@ -1072,6 +1127,9 @@ } out.writeVLong(field.sumDocFreq); out.writeVInt(field.docCount); + if (TERMS_VERSION_CURRENT >= TERMS_VERSION_META_ARRAY) { + out.writeVInt(field.longsSize); + } indexOut.writeVLong(field.indexStartFP); } writeTrailer(out, dirStart); diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java 2013-09-01 19:36:45.498625704 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java 2013-08-23 23:36:52.559772427 +0800 @@ -32,6 +32,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -121,11 +122,6 @@ long proxOffset; long skipOffset; - // Only used by the "primary" TermState -- clones don't - // copy this (basically they are "transient"): - ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...? - byte[] bytes; - @Override public StandardTermState clone() { StandardTermState other = new StandardTermState(); @@ -140,11 +136,6 @@ freqOffset = other.freqOffset; proxOffset = other.proxOffset; skipOffset = other.skipOffset; - - // Do not copy bytes, bytesReader (else TermState is - // very heavy, ie drags around the entire block's - // byte[]). On seek back, if next() is in fact used - // (rare!), they will be re-read from disk. } @Override @@ -171,38 +162,18 @@ } } - /* Reads but does not decode the byte[] blob holding - metadata for the current terms block */ @Override - public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException { - final StandardTermState termState = (StandardTermState) _termState; - - final int len = termsIn.readVInt(); - - // if (DEBUG) System.out.println(" SPR.readTermsBlock bytes=" + len + " ts=" + _termState); - if (termState.bytes == null) { - termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; - termState.bytesReader = new ByteArrayDataInput(); - } else if (termState.bytes.length < len) { - termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; - } - - termsIn.readBytes(termState.bytes, 0, len); - termState.bytesReader.reset(termState.bytes, 0, len); - } - - @Override - public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) + public void decodeTerm(long[] longs, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute) throws IOException { final StandardTermState termState = (StandardTermState) _termState; // if (DEBUG) System.out.println("SPR: nextTerm seg=" + segment + " tbOrd=" + termState.termBlockOrd + " bytesReader.fp=" + termState.bytesReader.getPosition()); final boolean isFirstTerm = termState.termBlockOrd == 0; - - if (isFirstTerm) { - termState.freqOffset = termState.bytesReader.readVLong(); - } else { - termState.freqOffset += termState.bytesReader.readVLong(); + if (absolute) { + termState.freqOffset = 0; + termState.proxOffset = 0; } + + termState.freqOffset += in.readVLong(); /* if (DEBUG) { System.out.println(" dF=" + termState.docFreq); @@ -212,7 +183,7 @@ assert termState.freqOffset < freqIn.length(); if (termState.docFreq >= skipMinimum) { - termState.skipOffset = termState.bytesReader.readVLong(); + termState.skipOffset = in.readVLong(); // if (DEBUG) System.out.println(" skipOffset=" + termState.skipOffset + " vs freqIn.length=" + freqIn.length()); assert termState.freqOffset + termState.skipOffset < freqIn.length(); } else { @@ -220,11 +191,7 @@ } if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) { - if (isFirstTerm) { - termState.proxOffset = termState.bytesReader.readVLong(); - } else { - termState.proxOffset += termState.bytesReader.readVLong(); - } + termState.proxOffset += in.readVLong(); // if (DEBUG) System.out.println(" proxFP=" + termState.proxOffset); } } diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java 2013-09-01 19:36:45.548625704 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java 2013-09-02 00:34:44.635086876 +0800 @@ -44,7 +44,7 @@ * Upper limit of the number of bytes that might be required to stored * BLOCK_SIZE encoded values. */ - static final int MAX_ENCODED_SIZE = BLOCK_SIZE * 4; + public static final int MAX_ENCODED_SIZE = BLOCK_SIZE * 4; /** * Upper limit of the number of values that might be decoded in a single call to @@ -52,7 +52,7 @@ * BLOCK_SIZE are garbage, it is necessary to allocate value buffers * whose size is >= MAX_DATA_SIZE to avoid {@link ArrayIndexOutOfBoundsException}s. */ - static final int MAX_DATA_SIZE; + public static final int MAX_DATA_SIZE; static { int maxDataSize = 0; for(int version=PackedInts.VERSION_START;version<=PackedInts.VERSION_CURRENT;version++) { @@ -96,7 +96,7 @@ /** * Create a new {@link ForUtil} instance and save state into out. */ - ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { + public ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException { out.writeVInt(PackedInts.VERSION_CURRENT); encodedSizes = new int[33]; encoders = new PackedInts.Encoder[33]; @@ -122,7 +122,7 @@ /** * Restore a {@link ForUtil} from a {@link DataInput}. */ - ForUtil(DataInput in) throws IOException { + public ForUtil(DataInput in) throws IOException { int packedIntsVersion = in.readVInt(); PackedInts.checkVersion(packedIntsVersion); encodedSizes = new int[33]; @@ -154,7 +154,7 @@ * @param out the destination output * @throws IOException If there is a low-level I/O error */ - void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { + public void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { if (isAllEqual(data)) { out.writeByte((byte) ALL_VALUES_EQUAL); out.writeVInt(data[0]); @@ -183,7 +183,7 @@ * @param decoded where to write decoded data * @throws IOException If there is a low-level I/O error */ - void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { + public void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException { final int numBits = in.readByte(); assert numBits <= 32 : numBits; @@ -209,7 +209,7 @@ * @param in the input where to read data * @throws IOException If there is a low-level I/O error */ - void skipBlock(IndexInput in) throws IOException { + public void skipBlock(IndexInput in) throws IOException { final int numBits = in.readByte(); if (numBits == ALL_VALUES_EQUAL) { in.readVInt(); diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java 2013-09-01 19:36:45.548625704 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java 2013-08-15 23:41:40.000000000 +0800 @@ -161,7 +161,7 @@ *
  • SkipFPDelta determines the position of this term's SkipData within the .doc * file. In particular, it is the length of the TermFreq data. * SkipDelta is only stored if DocFreq is not smaller than SkipMinimum - * (i.e. 128 in Lucene41PostingsFormat).
    • + * (i.e. 8 in Lucene41PostingsFormat). *
  • SingletonDocID is an optimization when a term only appears in one document. In this case, instead * of writing a file pointer to the .doc file (DocFPDelta), and then a VIntBlock at that location, the * single document ID is written to the term dictionary.
    • diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java 2013-09-01 19:36:45.551959037 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java 2013-09-01 10:08:11.702351174 +0800 @@ -59,6 +59,7 @@ private final IndexInput payIn; private final ForUtil forUtil; + private int version; // public static boolean DEBUG = false; @@ -71,27 +72,21 @@ try { docIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), ioContext); - CodecUtil.checkHeader(docIn, + version = CodecUtil.checkHeader(docIn, Lucene41PostingsWriter.DOC_CODEC, - Lucene41PostingsWriter.VERSION_CURRENT, + Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT); forUtil = new ForUtil(docIn); if (fieldInfos.hasProx()) { posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), ioContext); - CodecUtil.checkHeader(posIn, - Lucene41PostingsWriter.POS_CODEC, - Lucene41PostingsWriter.VERSION_CURRENT, - Lucene41PostingsWriter.VERSION_CURRENT); + CodecUtil.checkHeader(posIn, Lucene41PostingsWriter.POS_CODEC, version, version); if (fieldInfos.hasPayloads() || fieldInfos.hasOffsets()) { payIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), ioContext); - CodecUtil.checkHeader(payIn, - Lucene41PostingsWriter.PAY_CODEC, - Lucene41PostingsWriter.VERSION_CURRENT, - Lucene41PostingsWriter.VERSION_CURRENT); + CodecUtil.checkHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, version, version); } } @@ -111,7 +106,7 @@ // Make sure we are talking to the matching postings writer CodecUtil.checkHeader(termsIn, Lucene41PostingsWriter.TERMS_CODEC, - Lucene41PostingsWriter.VERSION_CURRENT, + Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT); final int indexBlockSize = termsIn.readVInt(); if (indexBlockSize != BLOCK_SIZE) { @@ -152,11 +147,6 @@ // freq is always implicitly totalTermFreq in this case. int singletonDocID; - // Only used by the "primary" TermState -- clones don't - // copy this (basically they are "transient"): - ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...? - byte[] bytes; - @Override public IntBlockTermState clone() { IntBlockTermState other = new IntBlockTermState(); @@ -174,11 +164,6 @@ lastPosBlockOffset = other.lastPosBlockOffset; skipOffset = other.skipOffset; singletonDocID = other.singletonDocID; - - // Do not copy bytes, bytesReader (else TermState is - // very heavy, ie drags around the entire block's - // byte[]). On seek back, if next() is in fact used - // (rare!), they will be re-read from disk. } @Override @@ -197,81 +182,69 @@ IOUtils.close(docIn, posIn, payIn); } - /* Reads but does not decode the byte[] blob holding - metadata for the current terms block */ @Override - public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException { - final IntBlockTermState termState = (IntBlockTermState) _termState; - - final int numBytes = termsIn.readVInt(); - - if (termState.bytes == null) { - termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; - termState.bytesReader = new ByteArrayDataInput(); - } else if (termState.bytes.length < numBytes) { - termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; - } - - termsIn.readBytes(termState.bytes, 0, numBytes); - termState.bytesReader.reset(termState.bytes, 0, numBytes); - } - - @Override - public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) + public void decodeTerm(long[] longs, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute) throws IOException { final IntBlockTermState termState = (IntBlockTermState) _termState; - final boolean isFirstTerm = termState.termBlockOrd == 0; final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; final boolean fieldHasPayloads = fieldInfo.hasPayloads(); - final DataInput in = termState.bytesReader; - if (isFirstTerm) { - if (termState.docFreq == 1) { - termState.singletonDocID = in.readVInt(); - termState.docStartFP = 0; - } else { - termState.singletonDocID = -1; - termState.docStartFP = in.readVLong(); + if (absolute) { + termState.docStartFP = 0; + termState.posStartFP = 0; + termState.payStartFP = 0; + } + if (version < Lucene41PostingsWriter.VERSION_META_ARRAY) { // impersonation + _decodeTerm(in, fieldInfo, termState); + return; + } + termState.docStartFP += longs[0]; + if (fieldHasPositions) { + termState.posStartFP += longs[1]; + if (fieldHasOffsets || fieldHasPayloads) { + termState.payStartFP += longs[2]; } - if (fieldHasPositions) { - termState.posStartFP = in.readVLong(); - if (termState.totalTermFreq > BLOCK_SIZE) { - termState.lastPosBlockOffset = in.readVLong(); - } else { - termState.lastPosBlockOffset = -1; - } - if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) { - termState.payStartFP = in.readVLong(); - } else { - termState.payStartFP = -1; - } + } + if (termState.docFreq == 1) { + termState.singletonDocID = in.readVInt(); + } else { + termState.singletonDocID = -1; + } + if (fieldHasPositions) { + if (termState.totalTermFreq > BLOCK_SIZE) { + termState.lastPosBlockOffset = in.readVLong(); + } else { + termState.lastPosBlockOffset = -1; } + } + if (termState.docFreq > BLOCK_SIZE) { + termState.skipOffset = in.readVLong(); + } else { + termState.skipOffset = -1; + } + } + private void _decodeTerm(DataInput in, FieldInfo fieldInfo, IntBlockTermState termState) throws IOException { + final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + final boolean fieldHasPayloads = fieldInfo.hasPayloads(); + if (termState.docFreq == 1) { + termState.singletonDocID = in.readVInt(); } else { - if (termState.docFreq == 1) { - termState.singletonDocID = in.readVInt(); + termState.singletonDocID = -1; + termState.docStartFP += in.readVLong(); + } + if (fieldHasPositions) { + termState.posStartFP += in.readVLong(); + if (termState.totalTermFreq > BLOCK_SIZE) { + termState.lastPosBlockOffset = in.readVLong(); } else { - termState.singletonDocID = -1; - termState.docStartFP += in.readVLong(); + termState.lastPosBlockOffset = -1; } - if (fieldHasPositions) { - termState.posStartFP += in.readVLong(); - if (termState.totalTermFreq > BLOCK_SIZE) { - termState.lastPosBlockOffset = in.readVLong(); - } else { - termState.lastPosBlockOffset = -1; - } - if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) { - long delta = in.readVLong(); - if (termState.payStartFP == -1) { - termState.payStartFP = delta; - } else { - termState.payStartFP += delta; - } - } + if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) { + termState.payStartFP += in.readVLong(); } } - if (termState.docFreq > BLOCK_SIZE) { termState.skipOffset = in.readVLong(); } else { diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java --- trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java 2013-09-01 19:36:45.548625704 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java 2013-09-03 19:43:02.246635047 +0800 @@ -25,14 +25,15 @@ import java.util.ArrayList; import java.util.List; +import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; -import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; @@ -65,13 +66,15 @@ // Increment version to change it final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; + final static int VERSION_META_ARRAY = 1; + //final static int VERSION_CURRENT = VERSION_START; + final static int VERSION_CURRENT = VERSION_META_ARRAY; final IndexOutput docOut; final IndexOutput posOut; final IndexOutput payOut; - private IndexOutput termsOut; + IntBlockTermState lastState; // How current field indexes postings: private boolean fieldHasFreqs; @@ -79,7 +82,7 @@ private boolean fieldHasOffsets; private boolean fieldHasPayloads; - // Holds starting file pointers for each term: + // Holds starting file pointers for current term: private long docTermStartFP; private long posTermStartFP; private long payTermStartFP; @@ -188,21 +191,51 @@ this(state, PackedInts.COMPACT); } + private final static class IntBlockTermState extends BlockTermState { + long docTermStartFP = 0; + long posTermStartFP = 0; + long payTermStartFP = 0; + long skipOffset = -1; + long lastPosBlockOffset = -1; + int singletonDocID = -1; + @Override + public String toString() { + return super.toString() + " docStartFP=" + docTermStartFP + " posStartFP=" + posTermStartFP + " payStartFP=" + payTermStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID; + } + } + + @Override + public IntBlockTermState newTermState() { + return new IntBlockTermState(); + } + @Override - public void start(IndexOutput termsOut) throws IOException { - this.termsOut = termsOut; + public void init(IndexOutput termsOut) throws IOException { CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT); termsOut.writeVInt(BLOCK_SIZE); } @Override - public void setField(FieldInfo fieldInfo) { + public int setField(FieldInfo fieldInfo) { IndexOptions indexOptions = fieldInfo.getIndexOptions(); fieldHasFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; fieldHasPayloads = fieldInfo.hasPayloads(); skipWriter.setField(fieldHasPositions, fieldHasOffsets, fieldHasPayloads); + lastState = newTermState(); + if (VERSION_CURRENT < VERSION_META_ARRAY) { + return 0; + } + if (fieldHasPositions) { + if (fieldHasPayloads || fieldHasOffsets) { + return 3; // doc + pos + pay FP + } else { + return 2; // doc + pos FP + } + } else { + return 1; // doc FP + } } @Override @@ -348,37 +381,18 @@ } } - private static class PendingTerm { - public final long docStartFP; - public final long posStartFP; - public final long payStartFP; - public final long skipOffset; - public final long lastPosBlockOffset; - public final int singletonDocID; - - public PendingTerm(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset, int singletonDocID) { - this.docStartFP = docStartFP; - this.posStartFP = posStartFP; - this.payStartFP = payStartFP; - this.skipOffset = skipOffset; - this.lastPosBlockOffset = lastPosBlockOffset; - this.singletonDocID = singletonDocID; - } - } - - private final List pendingTerms = new ArrayList(); - /** Called when we are done adding docs to this term */ @Override - public void finishTerm(TermStats stats) throws IOException { - assert stats.docFreq > 0; + public void finishTerm(BlockTermState _state) throws IOException { + IntBlockTermState state = (IntBlockTermState) _state; + assert state.docFreq > 0; // TODO: wasteful we are counting this (counting # docs // for this term) in two places? - assert stats.docFreq == docCount: stats.docFreq + " vs " + docCount; + assert state.docFreq == docCount: state.docFreq + " vs " + docCount; // if (DEBUG) { - // System.out.println("FPW.finishTerm docFreq=" + stats.docFreq); + // System.out.println("FPW.finishTerm docFreq=" + state.docFreq); // } // if (DEBUG) { @@ -389,7 +403,7 @@ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to it. final int singletonDocID; - if (stats.docFreq == 1) { + if (state.docFreq == 1) { // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq singletonDocID = docDeltaBuffer[0]; } else { @@ -420,8 +434,8 @@ // totalTermFreq is just total number of positions(or payloads, or offsets) // associated with current term. - assert stats.totalTermFreq != -1; - if (stats.totalTermFreq > BLOCK_SIZE) { + assert state.totalTermFreq != -1; + if (state.totalTermFreq > BLOCK_SIZE) { // record file offset for last pos in last block lastPosBlockOffset = posOut.getFilePointer() - posTermStartFP; } else { @@ -486,7 +500,7 @@ } } // if (DEBUG) { - // System.out.println(" totalTermFreq=" + stats.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset); + // System.out.println(" totalTermFreq=" + state.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset); // } } else { lastPosBlockOffset = -1; @@ -505,76 +519,80 @@ // System.out.println(" no skip: docCount=" + docCount); // } } - - long payStartFP; - if (stats.totalTermFreq >= BLOCK_SIZE) { - payStartFP = payTermStartFP; + if (VERSION_CURRENT >= VERSION_META_ARRAY || state.totalTermFreq >= BLOCK_SIZE) { + state.payTermStartFP = payTermStartFP; } else { - payStartFP = -1; + state.payTermStartFP = -1; } - // if (DEBUG) { // System.out.println(" payStartFP=" + payStartFP); // } - - pendingTerms.add(new PendingTerm(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset, singletonDocID)); + state.docTermStartFP = docTermStartFP; + state.posTermStartFP = posTermStartFP; + state.singletonDocID = singletonDocID; + state.skipOffset = skipOffset; + state.lastPosBlockOffset = lastPosBlockOffset; docBufferUpto = 0; posBufferUpto = 0; lastDocID = 0; docCount = 0; } - - private final RAMOutputStream bytesWriter = new RAMOutputStream(); - + @Override - public void flushTermsBlock(int start, int count) throws IOException { - - if (count == 0) { - termsOut.writeByte((byte) 0); + public void encodeTerm(long[] longs, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException { + IntBlockTermState state = (IntBlockTermState)_state; + if (absolute) { + lastState = newTermState(); + } + if (VERSION_CURRENT < VERSION_META_ARRAY) { // impersonation + _encodeTerm(out, fieldInfo, state); return; } - - assert start <= pendingTerms.size(); - assert count <= start; - - final int limit = pendingTerms.size() - start + count; - - long lastDocStartFP = 0; - long lastPosStartFP = 0; - long lastPayStartFP = 0; - for(int idx=limit-count; idx ").append(i).append("\n"); + } + for (int j = 0; j < sortedTransitions[i].length; j++) { + b.append(" ").append(i); + sortedTransitions[i][j].appendDot(b); + } + } + return b.append("}\n").toString(); + } } diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java branch3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java --- trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java 2013-09-01 19:36:45.048625710 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java 2013-08-31 07:07:06.836823489 +0800 @@ -233,16 +233,19 @@ StringBuilder b = new StringBuilder(); b.append("node=" + node); b.append(" target=" + target); - b.append(" label=" + label); - if (flag(BIT_LAST_ARC)) { - b.append(" last"); - } + b.append(" label=0x" + Integer.toHexString(label)); if (flag(BIT_FINAL_ARC)) { b.append(" final"); } + if (flag(BIT_LAST_ARC)) { + b.append(" last"); + } if (flag(BIT_TARGET_NEXT)) { b.append(" targetNext"); } + if (flag(BIT_STOP_NODE)) { + b.append(" stop"); + } if (flag(BIT_ARC_HAS_OUTPUT)) { b.append(" output=" + output); } @@ -834,6 +837,9 @@ if (emptyOutput != null) { arc.flags = BIT_FINAL_ARC | BIT_LAST_ARC; arc.nextFinalOutput = emptyOutput; + if (emptyOutput != NO_OUTPUT) { + arc.flags |= BIT_ARC_HAS_FINAL_OUTPUT; + } } else { arc.flags = BIT_LAST_ARC; arc.nextFinalOutput = NO_OUTPUT; diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java branch3069/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java --- trunk/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java 2013-09-01 19:36:45.045292377 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java 2013-08-15 23:41:40.000000000 +0800 @@ -68,7 +68,7 @@ } // hash code for an unfrozen node. This must be identical - // to the un-frozen case (below)!! + // to the frozen case (below)!! private long hash(Builder.UnCompiledNode node) { final int PRIME = 31; //System.out.println("hash unfrozen"); diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java branch3069/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java --- trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java 2013-09-01 19:36:45.045292377 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java 2013-08-15 23:41:40.000000000 +0800 @@ -40,7 +40,7 @@ // (new object per byte/char/int) if eg used during // analysis - /** Eg common("foo", "foobar") -> "foo" */ + /** Eg common("foobar", "food") -> "foo" */ public abstract T common(T output1, T output2); /** Eg subtract("foobar", "foo") -> "bar" */ diff -ruN -x .svn -x build trunk/lucene/core/src/java/org/apache/lucene/util/fst/Util.java branch3069/lucene/core/src/java/org/apache/lucene/util/fst/Util.java --- trunk/lucene/core/src/java/org/apache/lucene/util/fst/Util.java 2013-09-01 19:36:45.048625710 +0800 +++ branch3069/lucene/core/src/java/org/apache/lucene/util/fst/Util.java 2013-08-15 23:41:40.000000000 +0800 @@ -762,10 +762,11 @@ */ private static String printableLabel(int label) { if (label >= 0x20 && label <= 0x7d) { - return Character.toString((char) label); - } else { - return "0x" + Integer.toHexString(label); + if (label != 0x22 && label != 0x5c) { // " OR \ + return Character.toString((char) label); + } } + return "0x" + Integer.toHexString(label); } /** Just maps each UTF16 unit (char) to the ints in an diff -ruN -x .svn -x build trunk/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java branch3069/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java --- trunk/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java 2013-09-01 19:36:43.551959060 +0800 +++ branch3069/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java 2013-08-15 23:41:40.000000000 +0800 @@ -340,7 +340,6 @@ loc++; } while (loc < termsArray.length && !acceptTermsSet.contains(termsArray[loc])); } - assertNull(te.next()); } } @@ -769,6 +768,118 @@ assertNull(te.next()); r.close(); + dir.close(); + } + public void testIntersectStartTerm() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setMergePolicy(new LogDocMergePolicy()); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(newStringField("field", "abc", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + doc.add(newStringField("field", "abd", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + doc.add(newStringField("field", "acd", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + doc.add(newStringField("field", "bcd", Field.Store.NO)); + w.addDocument(doc); + + w.forceMerge(1); + DirectoryReader r = w.getReader(); + w.close(); + AtomicReader sub = getOnlySegmentReader(r); + Terms terms = sub.fields().terms("field"); + + Automaton automaton = new RegExp(".*d", RegExp.NONE).toAutomaton(); + CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); + TermsEnum te; + + // should seek to startTerm + te = terms.intersect(ca, new BytesRef("aad")); + assertEquals("abd", te.next().utf8ToString()); + assertEquals(1, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc()); + assertEquals("acd", te.next().utf8ToString()); + assertEquals(2, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc()); + assertEquals("bcd", te.next().utf8ToString()); + assertEquals(3, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc()); + assertNull(te.next()); + + // should fail to find ceil label on second arc, rewind + te = terms.intersect(ca, new BytesRef("add")); + assertEquals("bcd", te.next().utf8ToString()); + assertEquals(3, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc()); + assertNull(te.next()); + + // should reach end + te = terms.intersect(ca, new BytesRef("bcd")); + assertNull(te.next()); + te = terms.intersect(ca, new BytesRef("ddd")); + assertNull(te.next()); + + r.close(); + dir.close(); + } + + public void testIntersectEmptyString() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + iwc.setMergePolicy(new LogDocMergePolicy()); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(newStringField("field", "", Field.Store.NO)); + doc.add(newStringField("field", "abc", Field.Store.NO)); + w.addDocument(doc); + + doc = new Document(); + // add empty string to both documents, so that singletonDocID == -1. + // For a FST-based term dict, we'll expect to see the first arc is + // flaged with HAS_FINAL_OUTPUT + doc.add(newStringField("field", "abc", Field.Store.NO)); + doc.add(newStringField("field", "", Field.Store.NO)); + w.addDocument(doc); + + w.forceMerge(1); + DirectoryReader r = w.getReader(); + w.close(); + AtomicReader sub = getOnlySegmentReader(r); + Terms terms = sub.fields().terms("field"); + + Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); // accept ALL + CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); + + TermsEnum te = terms.intersect(ca, null); + DocsEnum de; + + assertEquals("", te.next().utf8ToString()); + de = te.docs(null, null, DocsEnum.FLAG_NONE); + assertEquals(0, de.nextDoc()); + assertEquals(1, de.nextDoc()); + + assertEquals("abc", te.next().utf8ToString()); + de = te.docs(null, null, DocsEnum.FLAG_NONE); + assertEquals(0, de.nextDoc()); + assertEquals(1, de.nextDoc()); + + assertNull(te.next()); + + // pass empty string + te = terms.intersect(ca, new BytesRef("")); + + assertEquals("abc", te.next().utf8ToString()); + de = te.docs(null, null, DocsEnum.FLAG_NONE); + assertEquals(0, de.nextDoc()); + assertEquals(1, de.nextDoc()); + + assertNull(te.next()); + + r.close(); dir.close(); } } diff -ruN -x .svn -x build trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java branch3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java --- trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java 2013-09-01 19:36:21.805292643 +0800 +++ branch3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java 2013-08-23 10:43:11.840306031 +0800 @@ -46,7 +46,7 @@ import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util._TestUtil; -@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"}) +@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "TempFixedIntBlock", "TempVariableIntBlock", "TempSep", "TempRandom"}) public class TestPostingsHighlighterRanking extends LuceneTestCase { /** * indexes a bunch of gibberish, and then highlights top(n). diff -ruN -x .svn -x build trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java branch3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java --- trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java 2013-09-01 19:36:22.361959303 +0800 +++ branch3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java 2013-08-23 23:36:52.493105762 +0800 @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; @@ -33,6 +34,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; @@ -67,7 +69,6 @@ */ final int maxSkipLevels = 10; final int totalNumDocs; - IndexOutput termsOut; IndexOptions indexOptions; boolean storePayloads; @@ -81,6 +82,9 @@ int lastPosition; int lastOffset; + final static StandardTermState emptyState = new StandardTermState(); + StandardTermState lastState; + // private String segment; /** Creates a {@link Lucene40PostingsWriter}, with the @@ -134,8 +138,7 @@ } @Override - public void start(IndexOutput termsOut) throws IOException { - this.termsOut = termsOut; + public void init(IndexOutput termsOut) throws IOException { CodecUtil.writeHeader(termsOut, Lucene40PostingsReader.TERMS_CODEC, Lucene40PostingsReader.VERSION_CURRENT); termsOut.writeInt(skipInterval); // write skipInterval termsOut.writeInt(maxSkipLevels); // write maxSkipLevels @@ -143,6 +146,12 @@ } @Override + public BlockTermState newTermState() { + return new StandardTermState(); + } + + + @Override public void startTerm() { freqStart = freqOut.getFilePointer(); //if (DEBUG) System.out.println("SPW: startTerm freqOut.fp=" + freqStart); @@ -159,7 +168,7 @@ // Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes @Override - public void setField(FieldInfo fieldInfo) { + public int setField(FieldInfo fieldInfo) { //System.out.println("SPW: setField"); /* if (BlockTreeTermsWriter.DEBUG && fieldInfo.name.equals("id")) { @@ -173,8 +182,10 @@ storeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; storePayloads = fieldInfo.hasPayloads(); + lastState = emptyState; //System.out.println(" set init blockFreqStart=" + freqStart); //System.out.println(" set init blockProxStart=" + proxStart); + return 0; } int lastDocID; @@ -265,94 +276,48 @@ public void finishDoc() { } - private static class PendingTerm { - public final long freqStart; - public final long proxStart; - public final long skipOffset; - - public PendingTerm(long freqStart, long proxStart, long skipOffset) { - this.freqStart = freqStart; - this.proxStart = proxStart; - this.skipOffset = skipOffset; - } + private static class StandardTermState extends BlockTermState { + public long freqStart; + public long proxStart; + public long skipOffset; } - private final List pendingTerms = new ArrayList(); - /** Called when we are done adding docs to this term */ @Override - public void finishTerm(TermStats stats) throws IOException { - + public void finishTerm(BlockTermState _state) throws IOException { + StandardTermState state = (StandardTermState)_state; // if (DEBUG) System.out.println("SPW: finishTerm seg=" + segment + " freqStart=" + freqStart); - assert stats.docFreq > 0; + assert state.docFreq > 0; // TODO: wasteful we are counting this (counting # docs // for this term) in two places? - assert stats.docFreq == df; - - final long skipOffset; + assert state.docFreq == df; + state.freqStart = freqStart; + state.proxStart = proxStart; if (df >= skipMinimum) { - skipOffset = skipListWriter.writeSkip(freqOut)-freqStart; + state.skipOffset = skipListWriter.writeSkip(freqOut)-freqStart; } else { - skipOffset = -1; + state.skipOffset = -1; } - - pendingTerms.add(new PendingTerm(freqStart, proxStart, skipOffset)); - lastDocID = 0; df = 0; } - private final RAMOutputStream bytesWriter = new RAMOutputStream(); - @Override - public void flushTermsBlock(int start, int count) throws IOException { - //if (DEBUG) System.out.println("SPW: flushTermsBlock start=" + start + " count=" + count + " left=" + (pendingTerms.size()-count) + " pendingTerms.size()=" + pendingTerms.size()); - - if (count == 0) { - termsOut.writeByte((byte) 0); - return; - } - - assert start <= pendingTerms.size(); - assert count <= start; - - final int limit = pendingTerms.size() - start + count; - final PendingTerm firstTerm = pendingTerms.get(limit - count); - // First term in block is abs coded: - bytesWriter.writeVLong(firstTerm.freqStart); - - if (firstTerm.skipOffset != -1) { - assert firstTerm.skipOffset > 0; - bytesWriter.writeVLong(firstTerm.skipOffset); + public void encodeTerm(long[] empty, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException { + StandardTermState state = (StandardTermState)_state; + if (absolute) { + lastState = emptyState; + } + out.writeVLong(state.freqStart - lastState.freqStart); + if (state.skipOffset != -1) { + assert state.skipOffset > 0; + out.writeVLong(state.skipOffset); } if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) { - bytesWriter.writeVLong(firstTerm.proxStart); + out.writeVLong(state.proxStart - lastState.proxStart); } - long lastFreqStart = firstTerm.freqStart; - long lastProxStart = firstTerm.proxStart; - for(int idx=limit-count+1; idx