diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsReaderBase.java b/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsReaderBase.java index 690b744..1dbffb7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsReaderBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsReaderBase.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.Bits; import org.apache.lucene.codecs.temp.TempTermState; @@ -52,11 +53,8 @@ public abstract class TempPostingsReaderBase implements Closeable { * dictionary {@link IndexInput}. */ public abstract void init(IndexInput termsIn) throws IOException; - /** Return a newly created empty TermState */ - public abstract TempTermState newTermState() throws IOException; - - /** Actually decode metadata for next term */ - public abstract void nextTerm(FieldInfo fieldInfo, TempTermState state) throws IOException; + /** Return a newly created empty TermMetaData */ + public abstract TermMetaData newTermMetaData() throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ @@ -69,9 +67,4 @@ public abstract class TempPostingsReaderBase implements Closeable { @Override public abstract void close() throws IOException; - - /** Reads data for all terms in the next block; this - * method should merely load the byte[] blob but not - * decode, which is done in {@link #nextTerm}. */ - public abstract void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, TempTermState termState) throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsWriterBase.java index ef41ea3..d691d6f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TempPostingsWriterBase.java @@ -22,6 +22,7 @@ import java.io.Closeable; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.codecs.temp.TempTermState; /** * Extension of {@link PostingsConsumer} to support pluggable term dictionaries. @@ -50,20 +51,17 @@ public abstract class TempPostingsWriterBase extends PostingsConsumer implements * the provided {@code termsOut}. */ public abstract void start(IndexOutput termsOut) throws IOException; + /** Return a newly created empty TermMetaData*/ + public abstract TermMetaData newTermMetaData() throws IOException; + /** Start a new term. Note that a matching call to {@link - * #finishTerm(TermStats)} is done, only if the term has at least one + * #finishTerm(TempTermState)} is done, only if the term has at least one * document. */ public abstract void startTerm() throws IOException; - /** Flush count terms starting at start "backwards", as a - * block. start is a negative offset from the end of the - * terms stack, ie bigger start means further back in - * the stack. */ - public abstract void flushTermsBlock(int start, int count) throws IOException; - /** Finishes the current term. The provided {@link - * TermStats} contains the term's summary statistics. */ - public abstract void finishTerm(TermStats stats) throws IOException; + * TempTermState} contains the term's summary statistics. */ + public abstract void finishTerm(TempTermState state) throws IOException; /** Called when the writing switches to another field. */ public abstract void setField(FieldInfo fieldInfo); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermMetaData.java b/lucene/core/src/java/org/apache/lucene/codecs/TermMetaData.java new file mode 100644 index 0000000..1021e99 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermMetaData.java @@ -0,0 +1,66 @@ +package org.apache.lucene.codecs; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.codecs.temp.TempTermState; + +public abstract class TermMetaData implements Cloneable { + + /* no arg means the instance will be always 'less than' any other instance */ + public TermMetaData() { + } + + public TermMetaData clone() { + try { + return (TermMetaData)super.clone(); + } catch (CloneNotSupportedException cnse) { + throw new RuntimeException(cnse); + } + } + public abstract void copyFrom(TermMetaData other); + + /* return a cloned instance, + * for monotonic fields, the values are (this - dec) */ + public abstract TermMetaData subtract(TermMetaData dec); + + /* return a cloned instance, + * for monotonic fields, the values are (this + inc) */ + public abstract TermMetaData add(TermMetaData inc); + + /* return current instance, + * for 'empty' fields in 'this', pad with values from its precedent + * e.g. in usual cases, value of an 'empty' field is -1 */ + public abstract TermMetaData pad(TermMetaData pre); + + /* return current instance, + * clear all fields so that this.equals(TermMetaData()) */ + public abstract TermMetaData clear(); + + public abstract void write(DataOutput out, FieldInfo info, TempTermState state) throws IOException; + + public abstract void read(DataInput out, FieldInfo info, TempTermState state) throws IOException; + + public String toString() { + return "TermMetaData"; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermProtoData.java b/lucene/core/src/java/org/apache/lucene/codecs/TermProtoData.java deleted file mode 100644 index 20fad44..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermProtoData.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.apache.lucene.codecs; - -import java.io.IOException; - - -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LongsRef; - -public class TermProtoData implements Comparable { - // TermProtoData consists of two parts: - // - // The base part: - // long array, in which every value increases - // monotonically at the same time. - // - // The extend part: - // byte array, in which non-monotonical values - // are stored/encoded. - // - // NOTE: For raw output, - // it is always assumed that, when we have - // this.base[i] < another.base[i], - // for all j in [base.offset, base.end) - // this.base[j] <= another.base[j] - // with this property, we might have better compression - // for base part. - // - // However, this property is not guraranteed for all intermediate - // outputs in a FST, e.g. a TermProtoData shared by two arcs might - // get a 'skewed' output, which is not possible to be compared with others - // Therefore during building phase, we have to iterate each long value - // to see whether the 'comparable' property still holds. - // - // NOTE: only use signed part of long value, which is 63 bits - // - public final LongsRef base; - public final BytesRef extend; - - public TermProtoData() { - this.base = null; - this.extend = null; - } - - public TermProtoData(LongsRef l, BytesRef b) { - this.base = l; - this.extend = b; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } else if (other instanceof TermProtoData) { - return this.base.equals(((TermProtoData)other).base); - } else { - return false; - } - } - - @Override - public int compareTo(TermProtoData other) { - // nocommit: just lazy to write the full codes: - // to use this, we should make sure it is - // really comparable first - return this.base.compareTo(other.base); - } - - @Override - public String toString() { - return base.toString() + " " + extend.toString(); - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermProtoOutputs.java b/lucene/core/src/java/org/apache/lucene/codecs/TermProtoOutputs.java deleted file mode 100644 index e765759..0000000 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermProtoOutputs.java +++ /dev/null @@ -1,171 +0,0 @@ -package org.apache.lucene.codecs; - -import java.io.IOException; - -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.fst.Outputs; -import org.apache.lucene.util.LongsRef; - -public class TermProtoOutputs extends Outputs { - private final static TermProtoData NO_OUTPUT = new TermProtoData(); - private final static TermProtoOutputs singleton = new TermProtoOutputs(); - - private TermProtoOutputs() { - } - - public static TermProtoOutputs getSingleton() { - return singleton; - } - - @Override - // - // Since base blob is fixed length, when these two are 'comparable' - // i.e. every value in LongsRef fits the same ordering, the smaller one will - // be the common. - // - // NOTE: the problem with TermProtoData is: - // we cannot usually reuse the whole TermProtoData like IntSequenceOutputs, - // since the extra blob of BytesRef restricts the possibility of reusing ... - // Therefore, the equal check in other==this doesn't actually make sense. - // - // NOTE: this is another case where 'doShare = false' sometimes - // creates a smaller FST..., since BytesRef blob is seldom the same - // between two terms (hmm, might be the same for synonym terms?), - // even we can fully share the LongsRef blob, we still have to create - // an extra output for two subtracted outputs, which, increases the - // size of FST - // - // NOTE: only LongsRef are 'shared', i.e. after sharing common value, - // the output of smaller one will be a all-zero LongsRef with BytesRef blob. - // So we need a heuristic to compress this ... - // - // nocommit: Builder.add() doesn't immediatelly consumes the output data, - // which means, the IntsRefs after one add() should all be deeply copied - // instead of being reused. quite hairly to detect it here, so the caller - // must be careful about this. - // - public TermProtoData common(TermProtoData t1, TermProtoData t2) { - if (t1 == NO_OUTPUT || t2 == NO_OUTPUT) { - return NO_OUTPUT; - } - assert t1.base != null; - assert t2.base != null; - assert t1.base.length == t2.base.length; - - LongsRef base1 = t1.base, base2 = t2.base; - long[] longs1 = base1.longs, longs2 = base2.longs; - int pos1 = base1.offset, pos2 = base2.offset; - int end1 = pos1 + base1.length; - boolean order = true; - - while (pos1 < end1 && longs1[pos1] == longs2[pos2]) { - pos1++; - pos2++; - } - if (pos1 < end1) { - order = (longs1[pos1] > longs2[pos2]); - if (order) { - // check whether base1 >= base2 - while (pos1 < end1 && longs1[pos1] >= longs2[pos2]) { - pos1++; - pos2++; - } - } else { - // check whether base1 <= base2 - while (pos1 < end1 && longs1[pos1] <= longs2[pos2]) { - pos1++; - pos2++; - } - } - } else { - // equal, so we might check the extend part - if (t1.extend != null && t1.extend.equals(t2.extend)) { - return t1; - } else { - return new TermProtoData(base1, null); - } - } - if (pos1 < end1) { - // not legally comparable - return NO_OUTPUT; - } else if (order) { - return new TermProtoData(base1, null); - } else { - return new TermProtoData(base2, null); - } - } - - @Override - // nocommit: maybe we should update the javadoc about this? - // this *actually* always assume that t2 <= t1 before calling the method - public TermProtoData subtract(TermProtoData t1, TermProtoData t2) { - if (t2 == NO_OUTPUT) { - return t1; - } - assert t1.base != null; - assert t2.base != null; - - LongsRef base1 = t1.base, base2 = t2.base; - int pos1 = base1.offset, pos2 = base2.offset; - int end1 = pos1 + base1.length; - long[] share = new long[base1.length]; - int pos = 0; - - while (pos1 < end1) { - share[pos] = base1.longs[pos1] - base2.longs[pos2]; - assert(share[pos] >= 0); - pos++; - pos1++; - pos2++; - } - return new TermProtoData(new LongsRef(share, 0, pos), null); - } - - @Override - // nocommit: need to check all-zero case? - // so we can reuse one LongsRef - public TermProtoData add(TermProtoData t1, TermProtoData t2) { - if (t1 == NO_OUTPUT) { - return t2; - } else if (t2 == NO_OUTPUT) { - return t1; - } - assert t1.base != null; - assert t2.base != null; - - LongsRef base1 = t1.base, base2 = t2.base; - int pos1 = base1.offset, pos2 = base2.offset; - int end1 = pos1 + base1.length; - long[] accum = new long[base1.length]; - int pos = 0; - while (pos1 < end1) { - accum[pos] = base1.longs[pos1] + base2.longs[pos2]; - assert(accum[pos] >= 0); - pos++; - pos1++; - pos2++; - } - return NO_OUTPUT; - } - - @Override - public void write(TermProtoData data, DataOutput out) throws IOException { - throw new IllegalStateException("not implemented"); - } - - @Override - public TermProtoData read(DataInput in) throws IOException { - throw new IllegalStateException("not implemented"); - } - - @Override - public TermProtoData getNoOutput() { - return NO_OUTPUT; - } - - @Override - public String outputToString(TermProtoData data) { - return data.toString(); - } -} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java index 0386369..938d4c0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java @@ -242,8 +242,8 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { if (payloads) { tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length); } - for (int i = 0; i < bufferedIndex; i++) { - if (offsets) { + if (offsets) { + for (int i = 0; i < bufferedIndex; i++) { tvf.writeVInt(offsetStartBuffer[i] - lastOffset); tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]); lastOffset = offsetEndBuffer[i]; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java index 4838f7a..df7fe6f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java @@ -161,7 +161,7 @@ import org.apache.lucene.util.packed.PackedInts; *
  • SkipFPDelta determines the position of this term's SkipData within the .doc * file. In particular, it is the length of the TermFreq data. * SkipDelta is only stored if DocFreq is not smaller than SkipMinimum - * (i.e. 8 in Lucene41PostingsFormat).
    • + * (i.e. 128 in Lucene41PostingsFormat). *
  • SingletonDocID is an optimization when a term only appears in one document. In this case, instead * of writing a file pointer to the .doc file (DocFPDelta), and then a VIntBlock at that location, the * single document ID is written to the term dictionary.
    • diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermState.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermState.java new file mode 100644 index 0000000..b6a027d --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermState.java @@ -0,0 +1,47 @@ +package org.apache.lucene.codecs.temp; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.TermState; + +public class TempBlockTermState extends TempTermState { + /** the term's ord in the current block */ + public int termBlockOrd; + + /** Sole constructor. (For invocation by subclass + * constructors, typically implicit.) */ + protected TempBlockTermState() { + } + + public TempBlockTermState clone() { + TempBlockTermState other = (TempBlockTermState)super.clone(); + return other; + } + + @Override + public void copyFrom(TermState _other) { + assert _other instanceof TempBlockTermState : "can not copy from " + _other.getClass().getName(); + super.copyFrom(_other); + TempBlockTermState other = (TempBlockTermState) _other; + termBlockOrd = other.termBlockOrd; + } + + @Override + public String toString() { + return super.toString() + " termBlockOrd=" + termBlockOrd; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsReader.java index c11afb8..a4ed6f8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsReader.java @@ -58,6 +58,7 @@ import org.apache.lucene.util.fst.Util; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.TempPostingsReaderBase; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.TermMetaData; /** A block-based terms index and dictionary that assigns * terms to variable length blocks according to how they @@ -620,7 +621,7 @@ public class TempBlockTermsReader extends FieldsProducer { FST.Arc arc; - final TempTermState termState; + final TempBlockTermState termState; // Cumulative output so far BytesRef outputPrefix; @@ -628,9 +629,14 @@ public class TempBlockTermsReader extends FieldsProducer { private int startBytePos; private int suffix; + // buffer current block of term metadata + ByteArrayDataInput bytesReader; + byte[] bytes; + public Frame(int ord) throws IOException { this.ord = ord; - termState = postingsReader.newTermState(); + termState = new TempBlockTermState(); + termState.meta = postingsReader.newTermMetaData(); termState.totalTermFreq = -1; } @@ -729,8 +735,17 @@ public class TempBlockTermsReader extends FieldsProducer { termState.termBlockOrd = 0; nextEnt = 0; - - postingsReader.readTermsBlock(in, fieldInfo, termState); + + // metadata + numBytes = in.readVInt(); + if (bytes == null) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + bytesReader = new ByteArrayDataInput(); + } else if (bytes.length < numBytes) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + } + in.readBytes(bytes, 0, numBytes); + bytesReader.reset(bytes, 0, numBytes); if (!isLastInFloor) { // Sub-blocks of a single floor block are always @@ -779,6 +794,8 @@ public class TempBlockTermsReader extends FieldsProducer { return isLeafBlock ? nextEnt : termState.termBlockOrd; } + // TODO: this actually decodes two kinds of blocks: + // stats as well as metadata public void decodeMetaData() throws IOException { // lazily catch up on metadata decode: @@ -788,7 +805,11 @@ public class TempBlockTermsReader extends FieldsProducer { // We must set/incr state.termCount because // postings impl can look at this termState.termBlockOrd = metaDataUpto; - + + if (termState.termBlockOrd == 0) { + termState.meta.clear(); + } + // TODO: better API would be "jump straight to term=N"??? while (metaDataUpto < limit) { @@ -800,14 +821,16 @@ public class TempBlockTermsReader extends FieldsProducer { // TODO: if docFreq were bulk decoded we could // just skipN here: + + // decode stats termState.docFreq = statsReader.readVInt(); - //if (DEBUG) System.out.println(" dF=" + state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { termState.totalTermFreq = termState.docFreq + statsReader.readVLong(); - //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq); } - postingsReader.nextTerm(fieldInfo, termState); + // decode metadata + termState.meta.read(bytesReader, fieldInfo, termState); + metaDataUpto++; termState.termBlockOrd++; } @@ -2210,7 +2233,7 @@ public class TempBlockTermsReader extends FieldsProducer { // } assert clearEOF(); if (target.compareTo(term) != 0 || !termExists) { - assert otherState != null && otherState instanceof TempTermState; + assert otherState != null && otherState instanceof TempBlockTermState; currentFrame = staticFrame; currentFrame.state.copyFrom(otherState); term.copyBytes(target); @@ -2298,11 +2321,16 @@ public class TempBlockTermsReader extends FieldsProducer { // metaData int metaDataUpto; - final TempTermState state; + final TempBlockTermState state; + + // buffer current block of term metadata + ByteArrayDataInput bytesReader; + byte[] bytes; public Frame(int ord) throws IOException { this.ord = ord; - state = postingsReader.newTermState(); + state = new TempBlockTermState(); + state.meta = postingsReader.newTermMetaData(); state.totalTermFreq = -1; } @@ -2399,9 +2427,16 @@ public class TempBlockTermsReader extends FieldsProducer { nextEnt = 0; lastSubFP = -1; - // TODO: we could skip this if !hasTerms; but - // that's rare so won't help much - postingsReader.readTermsBlock(in, fieldInfo, state); + // metadata + numBytes = in.readVInt(); + if (bytes == null) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + bytesReader = new ByteArrayDataInput(); + } else if (bytes.length < numBytes) { + bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; + } + in.readBytes(bytes, 0, numBytes); + bytesReader.reset(bytes, 0, numBytes); // Sub-blocks of a single floor block are always // written one after another -- tail recurse: @@ -2590,6 +2625,10 @@ public class TempBlockTermsReader extends FieldsProducer { // We must set/incr state.termCount because // postings impl can look at this state.termBlockOrd = metaDataUpto; + + if (state.termBlockOrd == 0) { + state.meta.clear(); + } // TODO: better API would be "jump straight to term=N"??? while (metaDataUpto < limit) { @@ -2602,14 +2641,16 @@ public class TempBlockTermsReader extends FieldsProducer { // TODO: if docFreq were bulk decoded we could // just skipN here: + + // decode stats state.docFreq = statsReader.readVInt(); - //if (DEBUG) System.out.println(" dF=" + state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { state.totalTermFreq = state.docFreq + statsReader.readVLong(); - //if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq); } - postingsReader.nextTerm(fieldInfo, state); + // decode metadata + state.meta.read(bytesReader, fieldInfo, state); + metaDataUpto++; state.termBlockOrd++; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsWriter.java index bef5ee9..0dfa206 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempBlockTermsWriter.java @@ -47,6 +47,7 @@ import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.TermsConsumer; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.TermMetaData; /* TODO: @@ -937,7 +938,7 @@ public class TempBlockTermsWriter extends FieldsConsumer { bytesWriter2.reset(); // Have postings writer write block - postingsWriter.flushTermsBlock(futureTermCount+termCount, termCount); + flushTermsBlock(futureTermCount+termCount, termCount); // Remove slice replaced by block: slice.clear(); @@ -957,6 +958,42 @@ public class TempBlockTermsWriter extends FieldsConsumer { return new PendingBlock(prefix, startFP, termCount != 0, isFloor, floorLeadByte, subIndices); } + // nocommit-noise: ok we're abusing this naming + private final RAMOutputStream bytesWriter3 = new RAMOutputStream(); + + /** Flush count terms starting at start "backwards", as a + * block. start is a negative offset from the end of the + * terms stack, ie bigger start means further back in + * the stack. */ + void flushTermsBlock(int start, int count) throws IOException { + if (count == 0) { + out.writeByte((byte) 0); + return; + } + + assert start <= pendingMetaData.size(); + assert count <= start; + + final int limit = pendingMetaData.size() - start + count; + + TermMetaData upto, last, delta; + last = postingsWriter.newTermMetaData(); + for(int idx=limit-count; idx pendingMetaData = new ArrayList(); @Override public void finishTerm(BytesRef text, TermStats stats) throws IOException { @@ -1005,7 +1043,13 @@ public class TempBlockTermsWriter extends FieldsConsumer { blockBuilder.add(Util.toIntsRef(text, scratchIntsRef), noOutputs.getNoOutput()); pending.add(new PendingTerm(BytesRef.deepCopyOf(text), stats)); - postingsWriter.finishTerm(stats); + + TempTermState termState = new TempTermState(); + termState.docFreq = stats.docFreq; + termState.totalTermFreq = stats.totalTermFreq; + //termState.meta = postingsWriter.newTermMetaData(); + postingsWriter.finishTerm(termState); + pendingMetaData.add(termState.meta); numTerms++; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempMetaData.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempMetaData.java new file mode 100644 index 0000000..019c507 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempMetaData.java @@ -0,0 +1,211 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import static org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat.BLOCK_SIZE; + +import java.io.IOException; + +import org.apache.lucene.codecs.TermMetaData; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; + + +// Must keep final because we do non-standard clone +final class TempMetaData extends TermMetaData { + long docStartFP; + long posStartFP; + long payStartFP; + long skipOffset; + long lastPosBlockOffset; + // docid when there is a single pulsed posting, otherwise -1 + // freq is always implicitly totalTermFreq in this case. + int singletonDocID; + + /* no arg means the instance will be always 'less than' any other instance */ + TempMetaData() { + docStartFP = 0; + posStartFP = 0; + payStartFP = -1; + singletonDocID = -1; + } + + TempMetaData(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset, int singletonDocID) { + this.docStartFP = docStartFP; + this.posStartFP = posStartFP; + this.payStartFP = payStartFP; + this.skipOffset = skipOffset; + this.lastPosBlockOffset = lastPosBlockOffset; + this.singletonDocID = singletonDocID; + } + + /* delta-encoding, only works on monotonical part */ + public TermMetaData subtract(TermMetaData _dec) { + TempMetaData dec = (TempMetaData) _dec; + TempMetaData ret = (TempMetaData)super.clone(); + if (ret.singletonDocID != -1) { + ret.docStartFP = 0; + } else { + assert dec.docStartFP >= 0; + assert dec.docStartFP <= ret.docStartFP; + ret.docStartFP -= dec.docStartFP; + } + if (ret.posStartFP != -1) { + assert dec.posStartFP <= ret.posStartFP; + ret.posStartFP -= dec.posStartFP; + } + if (ret.payStartFP != -1 && dec.payStartFP != -1) { + assert dec.payStartFP <= ret.payStartFP; + ret.payStartFP -= dec.payStartFP; + } + return ret; + } + + @Override + // nocommit: maybe an 'addself' will be more commonly used + public TermMetaData add(TermMetaData _inc) { + TempMetaData inc = (TempMetaData) _inc; + TempMetaData ret = (TempMetaData)super.clone(); + if (ret.singletonDocID != -1) { + ret.docStartFP = inc.docStartFP; + } else { + ret.docStartFP += inc.docStartFP; + } + if (ret.posStartFP != -1) { + ret.posStartFP += inc.posStartFP; + } + if (ret.payStartFP != -1 && inc.payStartFP != -1) { + ret.payStartFP += inc.payStartFP; + } + return ret; + } + + @Override + public TermMetaData pad(TermMetaData _pre) { + TempMetaData pre = (TempMetaData) _pre; + TempMetaData ret = this; + if (ret.singletonDocID != -1) { + ret.docStartFP = pre.docStartFP; + } + if (pre.payStartFP != -1 && ret.payStartFP == -1) { + ret.payStartFP = pre.payStartFP; + } + return ret; + } + @Override + public TermMetaData clear() { + this.docStartFP = 0; + this.posStartFP = 0; + this.payStartFP = -1; + this.singletonDocID = -1; + return this; + } + + @Override + // nocommit: quite stupid we have to calculate indexOptions here, + // (but we have to calculate this in Lucenen41PSTR as well... + // maybe we need more has*() APIs from fieldinfo()?) + public void write(DataOutput out, FieldInfo info, TempTermState state) throws IOException { + final IndexOptions indexOptions = info.getIndexOptions(); + boolean fieldHasFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + boolean fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + boolean fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + boolean fieldHasPayloads = info.hasPayloads(); + + if (singletonDocID == -1) { + out.writeVLong(docStartFP); + } else { + out.writeVInt(singletonDocID); + } + + if (fieldHasPositions) { + out.writeVLong(posStartFP); + if (lastPosBlockOffset != -1) { + out.writeVLong(lastPosBlockOffset); + } + if ((fieldHasPayloads || fieldHasOffsets) && payStartFP != -1) { + out.writeVLong(payStartFP); + } + } + if (skipOffset != -1) { + out.writeVLong(skipOffset); + } + } + + @Override + public void read(DataInput in, FieldInfo info, TempTermState state) throws IOException { + final IndexOptions indexOptions = info.getIndexOptions(); + boolean fieldHasFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + boolean fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + boolean fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + boolean fieldHasPayloads = info.hasPayloads(); + + if (state.docFreq == 1) { + singletonDocID = in.readVInt(); + } else { + singletonDocID = -1; + docStartFP += in.readVLong(); + } + if (fieldHasPositions) { + posStartFP += in.readVLong(); + if (state.totalTermFreq > BLOCK_SIZE) { + lastPosBlockOffset = in.readVLong(); + } else { + lastPosBlockOffset = -1; + } + if ((fieldHasPayloads || fieldHasOffsets) && state.totalTermFreq >= BLOCK_SIZE) { + if (payStartFP != -1) { + payStartFP += in.readVLong(); + } else { + payStartFP = in.readVLong(); + } + } + } + if (state.docFreq > BLOCK_SIZE) { + skipOffset = in.readVLong(); + } else { + skipOffset = -1; + } + } + + @Override + public TempMetaData clone() { + TempMetaData other = new TempMetaData(); + other.copyFrom(this); + return other; + } + + @Override + public void copyFrom(TermMetaData _other) { + //super.copyFrom(_other); + TempMetaData other = (TempMetaData) _other; + docStartFP = other.docStartFP; + posStartFP = other.posStartFP; + payStartFP = other.payStartFP; + lastPosBlockOffset = other.lastPosBlockOffset; + skipOffset = other.skipOffset; + singletonDocID = other.singletonDocID; + } + + @Override + public String toString() { + return "docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java index 25a0978..6bf7ed3 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java @@ -43,6 +43,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.codecs.TermMetaData; import org.apache.lucene.codecs.lucene41.ForUtil; import org.apache.lucene.codecs.lucene41.Lucene41SkipReader; @@ -143,55 +144,9 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } } - // Must keep final because we do non-standard clone - private final static class IntBlockTermState extends TempTermState { - long docStartFP; - long posStartFP; - long payStartFP; - long skipOffset; - long lastPosBlockOffset; - // docid when there is a single pulsed posting, otherwise -1 - // freq is always implicitly totalTermFreq in this case. - int singletonDocID; - - // Only used by the "primary" TermState -- clones don't - // copy this (basically they are "transient"): - ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...? - byte[] bytes; - - @Override - public IntBlockTermState clone() { - IntBlockTermState other = new IntBlockTermState(); - other.copyFrom(this); - return other; - } - - @Override - public void copyFrom(TermState _other) { - super.copyFrom(_other); - IntBlockTermState other = (IntBlockTermState) _other; - docStartFP = other.docStartFP; - posStartFP = other.posStartFP; - payStartFP = other.payStartFP; - lastPosBlockOffset = other.lastPosBlockOffset; - skipOffset = other.skipOffset; - singletonDocID = other.singletonDocID; - - // Do not copy bytes, bytesReader (else TermState is - // very heavy, ie drags around the entire block's - // byte[]). On seek back, if next() is in fact used - // (rare!), they will be re-read from disk. - } - - @Override - public String toString() { - return super.toString() + " docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID; - } - } - @Override - public IntBlockTermState newTermState() { - return new IntBlockTermState(); + public TermMetaData newTermMetaData() { + return new TempMetaData(); } @Override @@ -199,88 +154,6 @@ public final class TempPostingsReader extends TempPostingsReaderBase { IOUtils.close(docIn, posIn, payIn); } - /* Reads but does not decode the byte[] blob holding - metadata for the current terms block */ - @Override - public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, TempTermState _termState) throws IOException { - final IntBlockTermState termState = (IntBlockTermState) _termState; - - final int numBytes = termsIn.readVInt(); - - if (termState.bytes == null) { - termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; - termState.bytesReader = new ByteArrayDataInput(); - } else if (termState.bytes.length < numBytes) { - termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)]; - } - - termsIn.readBytes(termState.bytes, 0, numBytes); - termState.bytesReader.reset(termState.bytes, 0, numBytes); - } - - @Override - public void nextTerm(FieldInfo fieldInfo, TempTermState _termState) - throws IOException { - final IntBlockTermState termState = (IntBlockTermState) _termState; - final boolean isFirstTerm = termState.termBlockOrd == 0; - final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; - final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - final boolean fieldHasPayloads = fieldInfo.hasPayloads(); - - final DataInput in = termState.bytesReader; - if (isFirstTerm) { - if (termState.docFreq == 1) { - termState.singletonDocID = in.readVInt(); - termState.docStartFP = 0; - } else { - termState.singletonDocID = -1; - termState.docStartFP = in.readVLong(); - } - if (fieldHasPositions) { - termState.posStartFP = in.readVLong(); - if (termState.totalTermFreq > BLOCK_SIZE) { - termState.lastPosBlockOffset = in.readVLong(); - } else { - termState.lastPosBlockOffset = -1; - } - if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) { - termState.payStartFP = in.readVLong(); - } else { - termState.payStartFP = -1; - } - } - } else { - if (termState.docFreq == 1) { - termState.singletonDocID = in.readVInt(); - } else { - termState.singletonDocID = -1; - termState.docStartFP += in.readVLong(); - } - if (fieldHasPositions) { - termState.posStartFP += in.readVLong(); - if (termState.totalTermFreq > BLOCK_SIZE) { - termState.lastPosBlockOffset = in.readVLong(); - } else { - termState.lastPosBlockOffset = -1; - } - if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) { - long delta = in.readVLong(); - if (termState.payStartFP == -1) { - termState.payStartFP = delta; - } else { - termState.payStartFP += delta; - } - } - } - } - - if (termState.docFreq > BLOCK_SIZE) { - termState.skipOffset = in.readVLong(); - } else { - termState.skipOffset = -1; - } - } - @Override public DocsEnum docs(FieldInfo fieldInfo, TempTermState termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException { BlockDocsEnum docsEnum; @@ -292,7 +165,7 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } else { docsEnum = new BlockDocsEnum(fieldInfo); } - return docsEnum.reset(liveDocs, (IntBlockTermState) termState, flags); + return docsEnum.reset(liveDocs, termState, flags); } // TODO: specialize to liveDocs vs not @@ -316,7 +189,7 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } else { docsAndPositionsEnum = new BlockDocsAndPositionsEnum(fieldInfo); } - return docsAndPositionsEnum.reset(liveDocs, (IntBlockTermState) termState); + return docsAndPositionsEnum.reset(liveDocs, termState); } else { EverythingEnum everythingEnum; if (reuse instanceof EverythingEnum) { @@ -327,7 +200,7 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } else { everythingEnum = new EverythingEnum(fieldInfo); } - return everythingEnum.reset(liveDocs, (IntBlockTermState) termState, flags); + return everythingEnum.reset(liveDocs, termState, flags); } } @@ -391,16 +264,17 @@ public final class TempPostingsReader extends TempPostingsReaderBase { indexHasPayloads == fieldInfo.hasPayloads(); } - public DocsEnum reset(Bits liveDocs, IntBlockTermState termState, int flags) throws IOException { + public DocsEnum reset(Bits liveDocs, TempTermState termState, int flags) throws IOException { + final TempMetaData meta = (TempMetaData) termState.meta; this.liveDocs = liveDocs; // if (DEBUG) { // System.out.println(" FPR.reset: termState=" + termState); // } docFreq = termState.docFreq; totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq; - docTermStartFP = termState.docStartFP; - skipOffset = termState.skipOffset; - singletonDocID = termState.singletonDocID; + docTermStartFP = meta.docStartFP; + skipOffset = meta.skipOffset; + singletonDocID = meta.singletonDocID; if (docFreq > 1) { if (docIn == null) { // lazy init @@ -687,18 +561,19 @@ public final class TempPostingsReader extends TempPostingsReaderBase { indexHasPayloads == fieldInfo.hasPayloads(); } - public DocsAndPositionsEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException { + public DocsAndPositionsEnum reset(Bits liveDocs, TempTermState termState) throws IOException { + final TempMetaData meta = (TempMetaData) termState.meta; this.liveDocs = liveDocs; // if (DEBUG) { // System.out.println(" FPR.reset: termState=" + termState); // } docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; - skipOffset = termState.skipOffset; totalTermFreq = termState.totalTermFreq; - singletonDocID = termState.singletonDocID; + docTermStartFP = meta.docStartFP; + posTermStartFP = meta.posStartFP; + payTermStartFP = meta.payStartFP; + skipOffset = meta.skipOffset; + singletonDocID = meta.singletonDocID; if (docFreq > 1) { if (docIn == null) { // lazy init @@ -713,7 +588,7 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } else if (termState.totalTermFreq == BLOCK_SIZE) { lastPosBlockFP = -1; } else { - lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; + lastPosBlockFP = posTermStartFP + meta.lastPosBlockOffset; } doc = -1; @@ -1144,18 +1019,19 @@ public final class TempPostingsReader extends TempPostingsReaderBase { indexHasPayloads == fieldInfo.hasPayloads(); } - public EverythingEnum reset(Bits liveDocs, IntBlockTermState termState, int flags) throws IOException { + public EverythingEnum reset(Bits liveDocs, TempTermState termState, int flags) throws IOException { + final TempMetaData meta = (TempMetaData) termState.meta; this.liveDocs = liveDocs; // if (DEBUG) { // System.out.println(" FPR.reset: termState=" + termState); // } docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; - skipOffset = termState.skipOffset; totalTermFreq = termState.totalTermFreq; - singletonDocID = termState.singletonDocID; + docTermStartFP = meta.docStartFP; + posTermStartFP = meta.posStartFP; + payTermStartFP = meta.payStartFP; + skipOffset = meta.skipOffset; + singletonDocID = meta.singletonDocID; if (docFreq > 1) { if (docIn == null) { // lazy init @@ -1171,7 +1047,7 @@ public final class TempPostingsReader extends TempPostingsReaderBase { } else if (termState.totalTermFreq == BLOCK_SIZE) { lastPosBlockFP = -1; } else { - lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; + lastPosBlockFP = posTermStartFP + meta.lastPosBlockOffset; } this.needsOffsets = (flags & DocsAndPositionsEnum.FLAG_OFFSETS) != 0; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java index 0d0d4bb..ba331e9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java @@ -41,6 +41,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.codecs.TermMetaData; /** @@ -73,8 +74,6 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { final IndexOutput posOut; final IndexOutput payOut; - private IndexOutput termsOut; - // How current field indexes postings: private boolean fieldHasFreqs; private boolean fieldHasPositions; @@ -192,7 +191,6 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { @Override public void start(IndexOutput termsOut) throws IOException { - this.termsOut = termsOut; CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT); termsOut.writeVInt(BLOCK_SIZE); } @@ -350,37 +348,22 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { } } - private static class PendingTerm { - public final long docStartFP; - public final long posStartFP; - public final long payStartFP; - public final long skipOffset; - public final long lastPosBlockOffset; - public final int singletonDocID; - - public PendingTerm(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset, int singletonDocID) { - this.docStartFP = docStartFP; - this.posStartFP = posStartFP; - this.payStartFP = payStartFP; - this.skipOffset = skipOffset; - this.lastPosBlockOffset = lastPosBlockOffset; - this.singletonDocID = singletonDocID; - } + @Override + public TermMetaData newTermMetaData() { + return new TempMetaData(); } - private final List pendingTerms = new ArrayList(); - /** Called when we are done adding docs to this term */ @Override - public void finishTerm(TermStats stats) throws IOException { - assert stats.docFreq > 0; + public void finishTerm(TempTermState state) throws IOException { + assert state.docFreq > 0; // TODO: wasteful we are counting this (counting # docs // for this term) in two places? - assert stats.docFreq == docCount: stats.docFreq + " vs " + docCount; + assert state.docFreq == docCount: state.docFreq + " vs " + docCount; // if (DEBUG) { - // System.out.println("FPW.finishTerm docFreq=" + stats.docFreq); + // System.out.println("FPW.finishTerm docFreq=" + state.docFreq); // } // if (DEBUG) { @@ -391,7 +374,7 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to it. final int singletonDocID; - if (stats.docFreq == 1) { + if (state.docFreq == 1) { // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq singletonDocID = docDeltaBuffer[0]; } else { @@ -422,8 +405,8 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { // totalTermFreq is just total number of positions(or payloads, or offsets) // associated with current term. - assert stats.totalTermFreq != -1; - if (stats.totalTermFreq > BLOCK_SIZE) { + assert state.totalTermFreq != -1; + if (state.totalTermFreq > BLOCK_SIZE) { // record file offset for last pos in last block lastPosBlockOffset = posOut.getFilePointer() - posTermStartFP; } else { @@ -488,7 +471,7 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { } } // if (DEBUG) { - // System.out.println(" totalTermFreq=" + stats.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset); + // System.out.println(" totalTermFreq=" + state.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset); // } } else { lastPosBlockOffset = -1; @@ -509,7 +492,7 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { } long payStartFP; - if (stats.totalTermFreq >= BLOCK_SIZE) { + if (state.totalTermFreq >= BLOCK_SIZE) { payStartFP = payTermStartFP; } else { payStartFP = -1; @@ -518,66 +501,14 @@ public final class TempPostingsWriter extends TempPostingsWriterBase { // if (DEBUG) { // System.out.println(" payStartFP=" + payStartFP); // } - - pendingTerms.add(new PendingTerm(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset, singletonDocID)); + + state.meta = new TempMetaData(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset, singletonDocID); docBufferUpto = 0; posBufferUpto = 0; lastDocID = 0; docCount = 0; } - private final RAMOutputStream bytesWriter = new RAMOutputStream(); - - @Override - public void flushTermsBlock(int start, int count) throws IOException { - - if (count == 0) { - termsOut.writeByte((byte) 0); - return; - } - - assert start <= pendingTerms.size(); - assert count <= start; - - final int limit = pendingTerms.size() - start + count; - - long lastDocStartFP = 0; - long lastPosStartFP = 0; - long lastPayStartFP = 0; - for(int idx=limit-count; idx