Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1493518) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1501811) @@ -1015,7 +1015,7 @@ // Don't proceed if picked Codec is in the list of illegal ones. final String format = _TestUtil.getPostingsFormat("f"); assumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", - (format.equals("SimpleText") || format.equals("Memory") || format.equals("Direct"))); + (format.equals("SimpleText") || format.equals("Memory") || format.equals("Direct") || format.equals("TempFST"))); Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, conf); Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java (revision 1493518) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsReader.java (revision 1501811) @@ -71,7 +71,7 @@ IndexInput posIn = null; IndexInput payIn = null; try { - docIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempPostingsFormat.DOC_EXTENSION), + docIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempBlockPostingsFormat.DOC_EXTENSION), ioContext); CodecUtil.checkHeader(docIn, TempPostingsWriter.DOC_CODEC, @@ -80,7 +80,7 @@ forUtil = new ForUtil(docIn); if (fieldInfos.hasProx()) { - posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempPostingsFormat.POS_EXTENSION), + posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempBlockPostingsFormat.POS_EXTENSION), ioContext); CodecUtil.checkHeader(posIn, TempPostingsWriter.POS_CODEC, @@ -88,7 +88,7 @@ TempPostingsWriter.VERSION_CURRENT); if (fieldInfos.hasPayloads() || fieldInfos.hasOffsets()) { - payIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempPostingsFormat.PAY_EXTENSION), + payIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, TempBlockPostingsFormat.PAY_EXTENSION), ioContext); CodecUtil.checkHeader(payIn, TempPostingsWriter.PAY_CODEC, Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java (revision 1501811) @@ -0,0 +1,366 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.File; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeMap; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.fst.BytesRefFSTEnum; +import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.TempPostingsReaderBase; +import org.apache.lucene.codecs.CodecUtil; + +public class TempFSTTermsReader extends FieldsProducer { + final TreeMap fields = new TreeMap(); + final TempPostingsReaderBase postingsReader; + final IndexInput in; + boolean DEBUG = false; + //String tmpname; + + public TempFSTTermsReader(SegmentReadState state, TempPostingsReaderBase postingsReader) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempFSTTermsWriter.TERMS_EXTENSION); + //tmpname = termsFileName; + + this.postingsReader = postingsReader; + this.in = state.directory.openInput(termsFileName, state.context); + + boolean success = false; + try { + readHeader(in); + this.postingsReader.init(in); + seekDir(in); + + final FieldInfos fieldInfos = state.fieldInfos; + final int numFields = in.readVInt(); + for (int i = 0; i < numFields; i++) { + int fieldNumber = in.readVInt(); + FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); + long numTerms = in.readVLong(); + long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); + long sumDocFreq = in.readVLong(); + int docCount = in.readVInt(); + int longsSize = in.readVInt(); + TermsReader current = new TermsReader(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize); + TermsReader previous = fields.put(fieldInfo.name, current); + checkFieldSummary(state.segmentInfo, current, previous); + } + success = true; + } finally { + if (!success) { + in.close(); + } + } + } + + private int readHeader(IndexInput in) throws IOException { + return CodecUtil.checkHeader(in, + TempFSTTermsWriter.TERMS_CODEC_NAME, + TempFSTTermsWriter.TERMS_VERSION_START, + TempFSTTermsWriter.TERMS_VERSION_CURRENT); + } + private void seekDir(IndexInput in) throws IOException { + in.seek(in.length() - 8); + in.seek(in.readLong()); + } + private void checkFieldSummary(SegmentInfo info, TermsReader field, TermsReader previous) throws IOException { + // #docs with field must be <= #docs + if (field.docCount < 0 || field.docCount > info.getDocCount()) { + throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")"); + } + // #postings must be >= #docs with field + if (field.sumDocFreq < field.docCount) { + throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + in + ")"); + } + // #positions must be >= #postings + if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) { + throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + in + ")"); + } + if (previous != null) { + throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.name + " (resource=" + in + ")"); + } + } + + @Override + public Iterator iterator() { + return Collections.unmodifiableSet(fields.keySet()).iterator(); + } + + @Override + public Terms terms(String field) throws IOException { + assert field != null; + return fields.get(field); + } + + @Override + public int size() { + return fields.size(); + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(in, postingsReader); + } finally { + fields.clear(); + } + } + + final class TermsReader extends Terms { + final FieldInfo fieldInfo; + final long numTerms; + final long sumTotalTermFreq; + final long sumDocFreq; + final int docCount; + final int longsSize; + final FST dict; + + TermsReader(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) throws IOException { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.sumTotalTermFreq = sumTotalTermFreq; + this.sumDocFreq = sumDocFreq; + this.docCount = docCount; + this.longsSize = longsSize; + this.dict = new FST(in, new TempTermOutputs(fieldInfo, longsSize)); + //PrintWriter pw = new PrintWriter(new File("../graphs/ohohoh."+tmpname+".xxx.txt")); + //Util.toDot(dict, pw, false, false); + //pw.close(); + } + + // nocommit: implement intersect + // nocommit: why do we need this comparator overridden again and again? + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public TermsEnum iterator(TermsEnum reuse) throws IOException { + return new SegmentTermsEnum(); + } + + @Override + public boolean hasOffsets() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + } + + @Override + public boolean hasPositions() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + } + + @Override + public boolean hasPayloads() { + return fieldInfo.hasPayloads(); + } + + @Override + public long size() { + return numTerms; + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + // Iterates through terms in this field + final class SegmentTermsEnum extends TermsEnum { + final BytesRefFSTEnum fstEnum; + + /* Current term, null when enum ends or unpositioned */ + BytesRef term; + + /* Current term stats + decoded metadata (customized by PBF) */ + final TempTermState state; + + /* Current term stats + undecoded metadata (long[] & byte[]) */ + TempTermOutputs.TempMetaData meta; + ByteArrayDataInput bytesReader; + + /* True when current term's metadata is decoded */ + boolean decoded; + + /* True when current enum is 'positioned' by seekExact(TermState) */ + boolean seekPending; + + SegmentTermsEnum() throws IOException { + this.fstEnum = new BytesRefFSTEnum(dict); + this.state = postingsReader.newTermState(); + this.bytesReader = new ByteArrayDataInput(); + this.term = null; + this.decoded = false; + this.seekPending = false; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public TermState termState() throws IOException { + decodeMetaData(); + return state.clone(); + } + + @Override + public BytesRef term() { + return term; + } + + @Override + public int docFreq() throws IOException { + return state.docFreq; + } + + @Override + public long totalTermFreq() throws IOException { + return state.totalTermFreq; + } + + // Let PBF decodes metadata from long[] and byte[] + private void decodeMetaData() throws IOException { + if (!decoded && !seekPending) { + if (meta.bytes != null) { + bytesReader.reset(meta.bytes, 0, meta.bytes.length); + } + postingsReader.decodeTerm(meta.longs, bytesReader, fieldInfo, state); + decoded = true; + } + } + + // Update current enum according to FSTEnum + private void updateEnum(final InputOutput pair) { + if (pair == null) { + term = null; + } else { + term = pair.input; + meta = pair.output; + state.docFreq = meta.docFreq; + state.totalTermFreq = meta.totalTermFreq; + } + decoded = false; + seekPending = false; + } + + // nocommit: reuse? + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + decodeMetaData(); + return postingsReader.docs(fieldInfo, state, liveDocs, reuse, flags); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { + return null; + } + decodeMetaData(); + return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); + } + + @Override + public BytesRef next() throws IOException { + if (seekPending) { // previously positioned, but termOutputs not fetched + seekPending = false; + if (seekCeil(term, false) != SeekStatus.FOUND) { + return term; + } + } + updateEnum(fstEnum.next()); + return term; + } + + @Override + public boolean seekExact(final BytesRef target, final boolean useCache) throws IOException { + updateEnum(fstEnum.seekExact(target)); + return term != null; + } + + // nocommit: when will we useCache? + @Override + public SeekStatus seekCeil(final BytesRef target, final boolean useCache) throws IOException { + updateEnum(fstEnum.seekCeil(target)); + if (term == null) { + return SeekStatus.END; + } else { + return term.equals(target) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND; + } + } + + // nocommit: this method doesn't act as 'seekExact' right? + @Override + public void seekExact(BytesRef target, TermState otherState) { + if (term == null || target.compareTo(term) != 0) { + state.copyFrom(otherState); + term = BytesRef.deepCopyOf(target); + seekPending = true; + } + } + + // nocommit: do we need this? + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + } + } +} Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempTermOutputs.java (revision 1501811) @@ -0,0 +1,312 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.LongsRef; + + +// NOTE: outputs should be per-field, since +// longsSize is fixed for each field +public class TempTermOutputs extends Outputs { + private final static TempMetaData NO_OUTPUT = new TempMetaData(); + private static boolean DEBUG = false; + private boolean hasPos; + private int longsSize; + + public static class TempMetaData { + long[] longs; + byte[] bytes; + int docFreq; + long totalTermFreq; + TempMetaData() { + this.longs = null; + this.bytes = null; + this.docFreq = 0; + this.totalTermFreq = -1; + } + TempMetaData(long[] longs, byte[] bytes, int docFreq, long totalTermFreq) { + this.longs = longs; + this.bytes = bytes; + this.docFreq = docFreq; + this.totalTermFreq = totalTermFreq; + } + @Override + public int hashCode() { + int hash = 0; + if (longs != null) { + final int end = longs.length; + for (int i = 0; i < end; i++) { + hash -= longs[i]; + } + } + if (bytes != null) { + hash = -hash; + final int end = bytes.length; + for (int i = 0; i < end; i++) { + hash += bytes[i]; + } + } + return hash; + } + public String toString() { + if (this == NO_OUTPUT) { + return "no_output"; + } + StringBuffer sb = new StringBuffer(); + if (longs != null) { + sb.append("[ "); + for (int i = 0; i < longs.length; i++) { + sb.append(longs[i]+" "); + } + sb.append("]"); + } else { + sb.append("null"); + } + if (bytes != null) { + sb.append(" [ "); + for (int i = 0; i < bytes.length; i++) { + sb.append(Integer.toHexString((int)bytes[i] & 0xff)+" "); + } + sb.append("]"); + } else { + sb.append(" null"); + } + sb.append(" "+docFreq); + sb.append(" "+totalTermFreq); + return sb.toString(); + } + } + + private TempTermOutputs() { + } + + protected TempTermOutputs(FieldInfo fieldInfo, int longsSize) { + this.hasPos = (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY); + this.longsSize = longsSize; + } + + @Override + // + // The return value will be the smaller one, when these two are + // 'comparable', i.e. every value in long[] fits the same ordering. + // + // NOTE: + // Only long[] part is 'shared' and pushed towards root. + // byte[] and term stats will be on deeper arcs. + // + public TempMetaData common(TempMetaData t1, TempMetaData t2) { + if (DEBUG) System.out.print("common("+t1+", "+t2+") = "); + if (t1 == NO_OUTPUT || t2 == NO_OUTPUT) { + if (DEBUG) System.out.println("ret:"+NO_OUTPUT); + return NO_OUTPUT; + } + assert t1.longs.length == t2.longs.length; + + long[] min = t1.longs, max = t2.longs; + int pos = 0; + TempMetaData ret; + + while (pos < longsSize && min[pos] == max[pos]) { + pos++; + } + if (pos < longsSize) { // unequal long[] + if (min[pos] > max[pos]) { + min = t2.longs; + max = t1.longs; + } + // check whether strictly smaller + while (pos < longsSize && min[pos] <= max[pos]) { + pos++; + } + if (pos < longsSize || allZero(min)) { // not comparable or all-zero + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(min, null, 0, -1); + } + } else { // equal long[] + if (statsEqual(t1, t2) && (t1.bytes == null || bytesEqual(t1, t2))) { + ret = t1; + } else if (allZero(min)) { + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(min, null, 0, -1); + } + } + if (DEBUG) System.out.println("ret:"+ret); + return ret; + } + + @Override + public TempMetaData subtract(TempMetaData t1, TempMetaData t2) { + if (DEBUG) System.out.print("subtract("+t1+", "+t2+") = "); + if (t2 == NO_OUTPUT) { + if (DEBUG) System.out.println("ret:"+t1); + return t1; + } + assert t1.longs.length == t2.longs.length; + + int pos = 0; + long diff = 0; + long[] share = new long[longsSize]; + + while (pos < longsSize) { + share[pos] = t1.longs[pos] - t2.longs[pos]; + diff += share[pos]; + pos++; + } + + TempMetaData ret; + if (diff == 0 && statsEqual(t1, t2) && bytesEqual(t1, t2)) { + ret = NO_OUTPUT; + } else { + ret = new TempMetaData(share, t1.bytes, t1.docFreq, t1.totalTermFreq); + } + if (DEBUG) System.out.println("ret:"+ret); + return ret; + } + + @Override + public TempMetaData add(TempMetaData t1, TempMetaData t2) { + if (DEBUG) System.out.print("add("+t1+", "+t2+") = "); + if (t1 == NO_OUTPUT) { + if (DEBUG) System.out.println("ret:"+t2); + return t2; + } else if (t2 == NO_OUTPUT) { + if (DEBUG) System.out.println("ret:"+t1); + return t1; + } + assert t1.longs.length == t2.longs.length; + + int pos = 0; + long[] accum = new long[longsSize]; + + while (pos < longsSize) { + accum[pos] = t1.longs[pos] + t2.longs[pos]; + pos++; + } + + TempMetaData ret; + if (t2.bytes != null || t2.docFreq > 0) { + ret = new TempMetaData(accum, t2.bytes, t2.docFreq, t2.totalTermFreq); + } else { + ret = new TempMetaData(accum, t1.bytes, t1.docFreq, t1.totalTermFreq); + } + if (DEBUG) System.out.println("ret:"+ret); + return ret; + } + + @Override + public void write(TempMetaData data, DataOutput out) throws IOException { + int bit0 = allZero(data.longs) ? 0 : 1; + int bit1 = ((data.bytes == null || data.bytes.length == 0) ? 0 : 1) << 1; + int bit2 = ((data.docFreq == 0) ? 0 : 1) << 2; + int bits = bit0 | bit1 | bit2; + if (bit1 > 0) { // determine extra length + if (data.bytes.length < 32) { + bits |= (data.bytes.length << 3); + out.writeByte((byte)bits); + } else { + out.writeByte((byte)bits); + out.writeVInt(data.bytes.length); + } + } else { + out.writeByte((byte)bits); + } + if (bit0 > 0) { // not all-zero case + for (int pos = 0; pos < longsSize; pos++) { + out.writeVLong(data.longs[pos]); + } + } + if (bit1 > 0) { // bytes exists + out.writeBytes(data.bytes, 0, data.bytes.length); + } + if (bit2 > 0) { // stats exist + out.writeVInt(data.docFreq); + if (hasPos) { + out.writeVLong(data.totalTermFreq - data.docFreq); + } + } + } + + @Override + public TempMetaData read(DataInput in) throws IOException { + long[] longs = new long[longsSize]; + byte[] bytes = null; + int docFreq = 0; + long totalTermFreq = -1; + int bits = in.readByte() & 0xff; + int bit0 = bits & 1; + int bit1 = bits & 2; + int bit2 = bits & 4; + int bytesSize = (bits >>> 3); + if (bit1 > 0 && bytesSize == 0) { // determine extra length + bytesSize = in.readVInt(); + } + if (bit0 > 0) { // not all-zero case + for (int pos = 0; pos < longsSize; pos++) { + longs[pos] = in.readVLong(); + } + } + if (bit1 > 0) { // bytes exists + bytes = new byte[bytesSize]; + in.readBytes(bytes, 0, bytesSize); + } + if (bit2 > 0) { // stats exist + docFreq = in.readVInt(); + if (hasPos) { + totalTermFreq = docFreq + in.readVLong(); + } + } + return new TempMetaData(longs, bytes, docFreq, totalTermFreq); + } + + @Override + public TempMetaData getNoOutput() { + return NO_OUTPUT; + } + + @Override + public String outputToString(TempMetaData data) { + return data.toString(); + } + + static boolean statsEqual(final TempMetaData t1, final TempMetaData t2) { + return t1.docFreq == t2.docFreq && t1.totalTermFreq == t2.totalTermFreq; + } + static boolean bytesEqual(final TempMetaData t1, final TempMetaData t2) { + return Arrays.equals(t1.bytes, t2.bytes); + } + static boolean allZero(final long[] l) { + for (int i = 0; i < l.length; i++) { + if (l[i] != 0) { + return false; + } + } + return true; + } +} Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTPostingsFormat.java (revision 1501811) @@ -0,0 +1,77 @@ +package org.apache.lucene.codecs.temp; + + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.TempPostingsReaderBase; +import org.apache.lucene.codecs.TempPostingsWriterBase; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.IOUtils; + +public final class TempFSTPostingsFormat extends PostingsFormat { + public TempFSTPostingsFormat() { + super("TempFST"); + } + + @Override + public String toString() { + return getName(); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + TempPostingsWriterBase postingsWriter = new TempPostingsWriter(state); + + boolean success = false; + try { + FieldsConsumer ret = new TempFSTTermsWriter(state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsWriter); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + TempPostingsReaderBase postingsReader = new TempPostingsReader(state.directory, + state.fieldInfos, + state.segmentInfo, + state.context, + state.segmentSuffix); + boolean success = false; + try { + FieldsProducer ret = new TempFSTTermsReader(state, postingsReader); + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(postingsReader); + } + } + } +} Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java (revision 1493518) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempPostingsWriter.java (revision 1501811) @@ -119,7 +119,7 @@ public TempPostingsWriter(SegmentWriteState state, float acceptableOverheadRatio) throws IOException { super(); - docOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempPostingsFormat.DOC_EXTENSION), + docOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempBlockPostingsFormat.DOC_EXTENSION), state.context); IndexOutput posOut = null; IndexOutput payOut = null; @@ -129,7 +129,7 @@ forUtil = new ForUtil(acceptableOverheadRatio, docOut); if (state.fieldInfos.hasProx()) { posDeltaBuffer = new int[MAX_DATA_SIZE]; - posOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempPostingsFormat.POS_EXTENSION), + posOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempBlockPostingsFormat.POS_EXTENSION), state.context); CodecUtil.writeHeader(posOut, POS_CODEC, VERSION_CURRENT); @@ -150,7 +150,7 @@ } if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) { - payOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempPostingsFormat.PAY_EXTENSION), + payOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TempBlockPostingsFormat.PAY_EXTENSION), state.context); CodecUtil.writeHeader(payOut, PAY_CODEC, VERSION_CURRENT); } Index: lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsWriter.java (revision 1501811) @@ -0,0 +1,198 @@ +package org.apache.lucene.codecs.temp; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.fst.Builder; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Util; +import org.apache.lucene.codecs.TempPostingsWriterBase; +import org.apache.lucene.codecs.PostingsConsumer; +import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.TermsConsumer; +import org.apache.lucene.codecs.TermStats; +import org.apache.lucene.codecs.CodecUtil; + +/** FST based term dict, all the metadata held + * as output of FST */ + +public class TempFSTTermsWriter extends FieldsConsumer { + static final String TERMS_EXTENSION = "tmp"; + static final String TERMS_CODEC_NAME = "FST_TERMS_DICT"; + public static final int TERMS_VERSION_START = 0; + public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START; + + final TempPostingsWriterBase postingsWriter; + final FieldInfos fieldInfos; + final IndexOutput out; + final List fields = new ArrayList(); + + public TempFSTTermsWriter(SegmentWriteState state, TempPostingsWriterBase postingsWriter) throws IOException { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION); + + this.postingsWriter = postingsWriter; + this.fieldInfos = state.fieldInfos; + this.out = state.directory.createOutput(termsFileName, state.context); + + // nocommit: why try catch here? not catching createOutput? + boolean success = false; + try { + writeHeader(out); + this.postingsWriter.start(out); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + private void writeHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT); + } + private void writeTrailer(IndexOutput out, long dirStart) throws IOException { + out.writeLong(dirStart); + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + return new TermsWriter(field); + } + + @Override + public void close() throws IOException { + IOException ioe = null; + try { + // write field summary + final long dirStart = out.getFilePointer(); + + out.writeVInt(fields.size()); + for (FieldMetaData field : fields) { + out.writeVInt(field.fieldInfo.number); + out.writeVLong(field.numTerms); + if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) { + out.writeVLong(field.sumTotalTermFreq); + } + out.writeVLong(field.sumDocFreq); + out.writeVInt(field.docCount); + out.writeVInt(field.longsSize); + field.dict.save(out); + } + writeTrailer(out, dirStart); + } catch (IOException ioe2) { + ioe = ioe2; + } finally { + IOUtils.closeWhileHandlingException(ioe, out, postingsWriter); + } + } + + private static class FieldMetaData { + public final FieldInfo fieldInfo; + public final long numTerms; + public final long sumTotalTermFreq; + public final long sumDocFreq; + public final int docCount; + public final int longsSize; + public final FST dict; + + public FieldMetaData(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST fst) { + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.sumTotalTermFreq = sumTotalTermFreq; + this.sumDocFreq = sumDocFreq; + this.docCount = docCount; + this.longsSize = longsSize; + this.dict = fst; + } + } + + final class TermsWriter extends TermsConsumer { + private final Builder builder; + private final TempTermOutputs outputs; + private final FieldInfo fieldInfo; + private final int longsSize; + private long numTerms; + + private final IntsRef scratchTerm = new IntsRef(); + private final RAMOutputStream statsWriter = new RAMOutputStream(); + private final RAMOutputStream metaWriter = new RAMOutputStream(); + + TermsWriter(FieldInfo fieldInfo) { + this.numTerms = 0; + this.fieldInfo = fieldInfo; + this.longsSize = postingsWriter.setField(fieldInfo); + this.outputs = new TempTermOutputs(fieldInfo, longsSize); + this.builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public PostingsConsumer startTerm(BytesRef text) throws IOException { + postingsWriter.startTerm(); + return postingsWriter; + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + // write term meta data into fst + final TempTermOutputs.TempMetaData meta = new TempTermOutputs.TempMetaData(); + meta.longs = new long[longsSize]; + meta.bytes = null; + meta.docFreq = stats.docFreq; + meta.totalTermFreq = stats.totalTermFreq; + postingsWriter.finishTerm(meta.longs, metaWriter, stats); + final int bytesSize = (int)metaWriter.getFilePointer(); + if (bytesSize > 0) { + meta.bytes = new byte[bytesSize]; + metaWriter.writeTo(meta.bytes, 0); + metaWriter.reset(); + } + //System.out.println("add term:<"+text.utf8ToString()+", "+meta+">"); + builder.add(Util.toIntsRef(text, scratchTerm), meta); + numTerms++; + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { + // save FST dict + if (numTerms > 0) { + final FST fst = builder.finish(); + //fst.dump(); + fields.add(new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, fst)); + } + } + } +} Index: lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (revision 1493518) +++ lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (revision 1501811) @@ -40,7 +40,7 @@ // (new object per byte/char/int) if eg used during // analysis - /** Eg common("foo", "foobar") -> "foo" */ + /** Eg common("foobar", "food") -> "foo" */ public abstract T common(T output1, T output2); /** Eg subtract("foobar", "foo") -> "bar" */ Index: lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java (revision 1493518) +++ lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java (revision 1501811) @@ -68,7 +68,7 @@ } // hash code for an unfrozen node. This must be identical - // to the un-frozen case (below)!! + // to the frozen case (below)!! private long hash(Builder.UnCompiledNode node) { final int PRIME = 31; //System.out.println("hash unfrozen"); Index: lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat =================================================================== --- lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (revision 1493518) +++ lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (revision 1501811) @@ -15,4 +15,5 @@ org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat -org.apache.lucene.codecs.temp.TempPostingsFormat +org.apache.lucene.codecs.temp.TempBlockPostingsFormat +org.apache.lucene.codecs.temp.TempFSTPostingsFormat