Index: org/apache/lucene/index/DocumentsWriter.java =================================================================== --- org/apache/lucene/index/DocumentsWriter.java (revision 664366) +++ org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -17,30 +17,31 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.PrintStream; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.index.IndexWriter.DocState; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Similarity; import org.apache.lucene.search.Weight; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.UnicodeUtil; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.HashMap; -import java.util.ArrayList; -import java.util.Map.Entry; -import java.text.NumberFormat; -import java.util.Collections; - /** * This class accepts multiple added documents and directly * writes a single segment file. It does this more @@ -1034,21 +1035,22 @@ /** Returns true if the caller (IndexWriter) should now * flush. */ - boolean addDocument(Document doc, Analyzer analyzer) + boolean addDocument(Document doc, Analyzer analyzer, DocState docState) throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, null); + return updateDocument(doc, analyzer, null, docState); } boolean updateDocument(Term t, Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, t); + return updateDocument(doc, analyzer, t, null); } - boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm) + boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm, DocState docState) throws CorruptIndexException, IOException { // This call is synchronized but fast final DocumentsWriterThreadState state = getThreadState(doc, delTerm); + if (docState != null) docState.docId = state.docID; try { boolean success = false; try { Index: org/apache/lucene/index/IndexWriter.java =================================================================== --- org/apache/lucene/index/IndexWriter.java (revision 664366) +++ org/apache/lucene/index/IndexWriter.java (working copy) @@ -1915,7 +1915,15 @@ public void addDocument(Document doc) throws CorruptIndexException, IOException { addDocument(doc, analyzer); } - + + public static class DocState { + public int docId; + } + + public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { + addDocument(doc, analyzer, null); + } + /** * Adds a document to this index, using the provided analyzer instead of the * value of {@link #getAnalyzer()}. If the document contains more than @@ -1929,13 +1937,13 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { + public void addDocument(Document doc, Analyzer analyzer, DocState docState) throws CorruptIndexException, IOException { ensureOpen(); boolean doFlush = false; boolean success = false; try { try { - doFlush = docWriter.addDocument(doc, analyzer); + doFlush = docWriter.addDocument(doc, analyzer, docState); success = true; } finally { if (!success) { Index: org/apache/lucene/index/TagBaseBlocksTermDocs.java =================================================================== --- org/apache/lucene/index/TagBaseBlocksTermDocs.java (revision 0) +++ org/apache/lucene/index/TagBaseBlocksTermDocs.java (revision 0) @@ -0,0 +1,186 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; + +public abstract class TagBaseBlocksTermDocs implements TermDocs { + private TagBlockTermDocs current; + private TagBlockTermDocs[] termDocsArray; + private TagBlockInfos blockInfos; + int pointer; + int base; + private BitVector deletedDocs; + int skipInterval; + int maxSkipLevels; + Term term; + IndexInput[] streams; + TagIndexSnapshot snapshot; + private int[] starts; + + public TagBaseBlocksTermDocs(TagIndexSnapshot snapshot, BitVector deletedDocs, int skipInterval, int maxSkipLevels) { + this.snapshot = snapshot; + starts = snapshot.tagIndex.starts; + this.deletedDocs = deletedDocs; + this.skipInterval = skipInterval; + this.maxSkipLevels = maxSkipLevels; + } + + public void close() throws IOException { + Set set = new HashSet(); + for (int x=0; x < streams.length; x++) { + set.add(streams[x]); + } + Iterator iterator = set.iterator(); + while (iterator.hasNext()) { + IndexInput input = (IndexInput)iterator.next(); + input.close(); + } + } + + public void seek(TagBlockInfos blockInfos, IndexInput stream, Term term) throws IOException { + IndexInput[] streams = new IndexInput[blockInfos.positions.length]; + for (int x=0; x < streams.length; x++) { + streams[x] = stream; + } + seek(blockInfos, streams, term); + } + + public void seek(TagBlockInfos blockInfos, IndexInput[] streams, Term term) throws IOException { + this.blockInfos = blockInfos; + this.streams = streams; + this.term = term; + } + + public int doc() { + return base + current.doc(); + } + + public boolean next() throws IOException { + for (;;) { + if (current != null && current.next()) { + return true; + } else if (pointer < termDocsArray.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return false; + } + } + } + + private TagBlockTermDocs termDocs(int i) throws IOException { + if (termDocsArray[i] == null) { + termDocsArray[i] = new TagBlockTermDocs(i, blockInfos, deletedDocs, streams[i], skipInterval, maxSkipLevels); + } + return termDocsArray[i]; + } + + public int read(final int[] docs, final int[] freqs) throws IOException { + while (true) { + while (current == null) { + if (pointer < blockInfos.positions.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return 0; + } + } + int end = current.read(docs, freqs); + if (end == 0) { + current = null; + } else { + final int b = base; + for (int i = 0; i < end; i++) + docs[i] += b; + return end; + } + } + } + + public boolean skipTo(int target) throws IOException { + for (;;) { + if (current != null && current.skipTo(target - base)) { + return true; + } else if (pointer < blockInfos.positions.length) { + // pointer = blockIndex(target, starts, blockInfos.length, pointer); + pointer = binarySearch(starts, target, pointer, blockInfos.positions.length - pointer); + base = starts[pointer]; + current = termDocs(pointer); + } else + return false; + } + } + + // from instantiatedindex + public static final int binarySearch(int[] array, int key, int offset, int length) { + if (length == 0) { + return -1 - offset; + } + int min = offset, max = offset + length - 1; + int minVal = array[min], maxVal = array[max]; + + int nPreviousSteps = 0; + + // Uncomment these two lines to get statistics about the average number of + // steps in the test report : + // totalCalls++; + for (;;) { + // totalSteps++; + + // be careful not to compute key - minVal, for there might be an integer + // overflow. + if (key <= minVal) + return key == minVal ? min : -1 - min; + if (key >= maxVal) + return key == maxVal ? max : -2 - max; + + assert min != max; + + int pivot; + // A typical binarySearch algorithm uses pivot = (min + max) / 2. + // The pivot we use here tries to be smarter and to choose a pivot close + // to the expectable location of the key. + // This reduces dramatically the number of steps needed to get to the key. + // However, it does not work well with a logaritmic distribution of + // values, for instance. + // When the key is not found quickly the smart way, we switch to the + // standard pivot. + if (nPreviousSteps > 2) { + pivot = (min + max) >> 1; + // stop increasing nPreviousSteps from now on + } else { + // NOTE: We cannot do the following operations in int precision, because + // there might be overflows. + // long operations are slower than float operations with the hardware + // this was tested on (intel core duo 2, JVM 1.6.0). + // Overall, using float proved to be the safest and fastest approach. + pivot = min + (int) ((key - (float) minVal) / (maxVal - (float) minVal) * (max - min)); + nPreviousSteps++; + } + + int pivotVal = array[pivot]; + + // NOTE: do not store key - pivotVal because of overflows + if (key > pivotVal) { + min = pivot + 1; + max--; + } else if (key == pivotVal) { + return pivot; + } else { + min++; + max = pivot - 1; + } + maxVal = array[max]; + minVal = array[min]; + } + } + + public int freq() { + return 1; + } +} Index: org/apache/lucene/index/TagBlockInfos.java =================================================================== --- org/apache/lucene/index/TagBlockInfos.java (revision 0) +++ org/apache/lucene/index/TagBlockInfos.java (revision 0) @@ -0,0 +1,61 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public class TagBlockInfos implements TagWriteable { + public long[] positions; + public boolean[] tlogs; + public int[] docFreqs; + public int termNum; + public boolean tlog; + + public TagBlockInfos() {} + + // TODO: record difs between numbers + public TagBlockInfos(int termNum, boolean tlog, long position, IndexInput indexInput, long pointerBase) throws IOException { + this.tlog = tlog; + indexInput.seek(position); + this.termNum = termNum; + //termNum = indexInput.readVInt(); + int num = indexInput.readVInt(); + positions = new long[num]; + tlogs = new boolean[num]; + docFreqs = new int[num]; + for (int x=0; x < num; x++) { + positions[x] = indexInput.readVLong() + pointerBase; + } + for (int x=0; x < num; x++) { + tlogs[x] = TagUtil.toBoolean(indexInput.readByte()); + } + for (int x=0; x < num; x++) { + docFreqs[x] = indexInput.readVInt(); + } + } + + public void write(IndexOutput output) throws IOException { + //output.writeVInt(termNum); + output.writeVInt(positions.length); + for (long p : positions) { + output.writeVLong(p); + } + for (boolean b : tlogs) { + output.writeByte(TagUtil.toByte(b)); + } + for (int df : docFreqs) { + output.writeVInt(df); + } + } + /** + public static TagBlockInfo[] load(boolean tlog, TagBlockInfos tagBlockInfos, IndexInput indexInput) throws IOException { + TagBlockInfo[] array = new TagBlockInfo[tagBlockInfos.positions.length]; + for (int x=0; x < tagBlockInfos.positions.length; x++) { + indexInput.seek(tagBlockInfos.positions[x]); + array[x] = new TagBlockInfo(tlog, indexInput); + } + return array; + } + **/ +} Index: org/apache/lucene/index/TagBlockKey.java =================================================================== --- org/apache/lucene/index/TagBlockKey.java (revision 0) +++ org/apache/lucene/index/TagBlockKey.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.lucene.index; + +public class TagBlockKey { + private int blockNum; + private Term term; + + public TagBlockKey(int blockNum, Term term) { + this.blockNum = blockNum; + this.term = term; + } +} Index: org/apache/lucene/index/TagBlockLRUCache.java =================================================================== --- org/apache/lucene/index/TagBlockLRUCache.java (revision 0) +++ org/apache/lucene/index/TagBlockLRUCache.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.index; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Cache based on bytes used + * + */ +// TODO: needs synchronization +public class TagBlockLRUCache { + private LinkedHashMap map; + private int maxSize; + + public TagBlockLRUCache() { + + } + + public void put(Term term, int blockNum, TagBlockWriter tagBlockWriter) { + int size = tagBlockWriter.getDocsSize() + tagBlockWriter.getSkipSize(); + removeOverSized(); + } + + private void removeOverSized() { + Iterator> iterator = map.entrySet().iterator(); + int csize = 0; + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + TagBlockWriter tbw = entry.getValue(); + int size = tbw.getDocsSize() + tbw.getSkipSize(); + csize += size; + if (csize > maxSize) { + iterator.remove(); // remove entries that go over maxsize + } + } + } + + public TagBlockWriter get(Term term, int blockNum) { + TagBlockKey key = new TagBlockKey(blockNum, term); + return map.get(key); + } +} Index: org/apache/lucene/index/TagBlockMerger.java =================================================================== --- org/apache/lucene/index/TagBlockMerger.java (revision 0) +++ org/apache/lucene/index/TagBlockMerger.java (revision 0) @@ -0,0 +1,68 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.OpenBitSet; + +/** + * Takes old block, updated docs, outputs new block + * + */ +public class TagBlockMerger { + private TagBlockTermDocs tagBlockTermDocs; + private int numDocs; + private int skipInterval; + private int maxSkipLevels; + TagBlockWriter tagBlockData; + int docFreq; + + public TagBlockMerger(TagBlockTermDocs tagBlockTermDocs, int numDocs, int skipInterval, int maxSkipLevels) { + this.tagBlockTermDocs = tagBlockTermDocs; + this.numDocs = numDocs; + this.skipInterval = skipInterval; + this.maxSkipLevels = maxSkipLevels; + } + + public int merge(boolean add, DocIdSet docs) throws IOException { + OpenBitSet newSet = new OpenBitSet(numDocs); + setExistingDocs(newSet); + if (add) { + DocIdSetIterator addsIterator = docs.iterator(); + while (addsIterator.next()) { + newSet.fastSet(addsIterator.doc()); + } + } else { + DocIdSetIterator deletesIterator = docs.iterator(); + while (deletesIterator.next()) { + newSet.fastClear(deletesIterator.doc()); + } + } + docFreq = (int)newSet.cardinality(); + tagBlockData = new TagBlockWriter(skipInterval, maxSkipLevels, numDocs); + DocIdSetIterator newSetIterator = newSet.iterator(); + while (newSetIterator.next()) { + tagBlockData.add(newSetIterator.doc()); + } + return docFreq; + } + + public void write(IndexOutput output) throws IOException { + tagBlockData.write(output); + } + + /** + * Load existing docs for the block + * + * @param newSet + * @throws IOException + */ + private void setExistingDocs(OpenBitSet newSet) throws IOException { + while (tagBlockTermDocs.next()) { + newSet.set(tagBlockTermDocs.doc()); + } + } +} Index: org/apache/lucene/index/TagBlocksData.java =================================================================== --- org/apache/lucene/index/TagBlocksData.java (revision 0) +++ org/apache/lucene/index/TagBlocksData.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; + +public class TagBlocksData { + private int[] docs; + private RAMOutputStream output; + private TagBlockWriter[] tagBlockWriters; + private int termNum; + private int[] docFreqs; + private long[] positions; + + public TagBlocksData(int termNum, int[] docs, int skipInterval, int maxSkipLevels, int[] starts, int[] numDocsPerBlock) throws IOException { + this.termNum = termNum; + this.docs = docs; + int pos = 0; + tagBlockWriters = new TagBlockWriter[numDocsPerBlock.length]; + docFreqs = new int[numDocsPerBlock.length]; + positions = new long[numDocsPerBlock.length]; + for (int doc : docs) { + while (pos < numDocsPerBlock.length) { + if (starts[pos] <= doc && doc < starts[pos + 1]) { + if (tagBlockWriters[pos] == null) + tagBlockWriters[pos] = new TagBlockWriter(skipInterval, maxSkipLevels, numDocsPerBlock[pos]); + tagBlockWriters[pos].add(doc - starts[pos]); // doc minus base + break; + } else { + pos++; + } + } + } + Arrays.fill(docFreqs, 0); + Arrays.fill(positions, 0); + for (int x=0; x < tagBlockWriters.length; x++) { + if (tagBlockWriters[x] != null) { + docFreqs[x] = tagBlockWriters[x].getDocFreq(); + positions[x] = output.getFilePointer(); + tagBlockWriters[x].write(output); + } + } + } + + public long[] getPositions() { + return positions; + } + + public int[] getDocFreqs() { + return docFreqs; + } + + public void write(IndexOutput io) throws IOException { + output.writeTo(io); + } + + public int length() { + return (int)output.length(); + } +} Index: org/apache/lucene/index/TagBlockSkipListReader.java =================================================================== --- org/apache/lucene/index/TagBlockSkipListReader.java (revision 0) +++ org/apache/lucene/index/TagBlockSkipListReader.java (revision 0) @@ -0,0 +1,44 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexInput; + +public class TagBlockSkipListReader extends MultiLevelSkipListReader { + private long[] pointers; + + private long lastPointer; + + TagBlockSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { + super(skipStream, maxSkipLevels, skipInterval); + pointers = new long[maxSkipLevels]; + } + + void init(long skipPointer, long basePointer, int df) { + super.init(skipPointer, df); + lastPointer = basePointer; + + Arrays.fill(pointers, basePointer); + } + + long getPointer() { + return lastPointer; + } + + protected void seekChild(int level) throws IOException { + super.seekChild(level); + pointers[level] = lastPointer; + } + + protected void setLastSkipData(int level) { + super.setLastSkipData(level); + lastPointer = pointers[level]; + } + + protected int readSkipData(int level, IndexInput skipStream) throws IOException { + int delta = skipStream.readVInt(); + pointers[level] += skipStream.readVInt(); + return delta; + } +} Index: org/apache/lucene/index/TagBlockSkipListWriter.java =================================================================== --- org/apache/lucene/index/TagBlockSkipListWriter.java (revision 0) +++ org/apache/lucene/index/TagBlockSkipListWriter.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexOutput; + +public class TagBlockSkipListWriter extends MultiLevelSkipListWriter { + private int[] lastSkipDoc; + private long[] lastSkipPointer; + + private IndexOutput freqOutput; + + private int curDoc; + private long curFreqPointer; + + TagBlockSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput) { + super(skipInterval, numberOfSkipLevels, docCount); + this.freqOutput = freqOutput; + + lastSkipDoc = new int[numberOfSkipLevels]; + lastSkipPointer = new long[numberOfSkipLevels]; + } + + void setSkipData(int doc) { + this.curDoc = doc; + this.curFreqPointer = freqOutput.getFilePointer(); + } + + protected void resetSkip() { + super.resetSkip(); + Arrays.fill(lastSkipDoc, 0); + Arrays.fill(lastSkipPointer, freqOutput.getFilePointer()); + } + + protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + skipBuffer.writeVInt((int) (curFreqPointer - lastSkipPointer[level])); + + lastSkipDoc[level] = curDoc; + // System.out.println("write doc at level " + level + ": " + curDoc); + + lastSkipPointer[level] = curFreqPointer; + } +} Index: org/apache/lucene/index/TagBlockTermDocs.java =================================================================== --- org/apache/lucene/index/TagBlockTermDocs.java (revision 0) +++ org/apache/lucene/index/TagBlockTermDocs.java (revision 0) @@ -0,0 +1,106 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; + +public class TagBlockTermDocs { + private FieldInfos fieldInfos; + private IndexInput stream; + private int doc; + protected int count; + private long basePointer; + private long skipPointer; + private BitVector deletedDocs; + protected int numDocs; + private TagBlockSkipListReader skipListReader; + boolean haveSkipped; + private int skipInterval; + private int maxSkipLevels; + private int blockNum; + + public TagBlockTermDocs(int blockNum, TagBlockInfos tagBlockInfos, BitVector deletedDocs, IndexInput stream, int skipInterval, int maxSkipLevels) throws IOException { + this.blockNum = blockNum; + this.deletedDocs = deletedDocs; + this.stream = stream; + this.skipInterval = skipInterval; + this.maxSkipLevels = maxSkipLevels; + seek(tagBlockInfos, blockNum); + } + + public final int doc() { + return doc; + } + + private void seek(TagBlockInfos tagBlockInfos, int blockNum) throws IOException { + count = 0; + doc = 0; + this.blockNum = blockNum; + numDocs = tagBlockInfos.docFreqs[blockNum]; + basePointer = tagBlockInfos.positions[blockNum]; + stream.seek(basePointer); + int skipOffset = stream.readVInt(); + skipPointer = skipOffset + basePointer; + + haveSkipped = false; + } + + public boolean skipTo(int target) throws IOException { + if (numDocs >= skipInterval) { + if (skipListReader == null) + skipListReader = new TagBlockSkipListReader((IndexInput) stream.clone(), maxSkipLevels, skipInterval); + if (!haveSkipped) { + skipListReader.init(skipPointer, basePointer, numDocs); + haveSkipped = true; + } + int newCount = skipListReader.skipTo(target); + if (newCount > count) { + // TODO: make sure pointer is relative stream, probably is not + stream.seek(skipListReader.getPointer()); + doc = skipListReader.getDoc(); + count = newCount; + } + } + do { + if (!next()) + return false; + } while (target > doc); + return true; + } + + public int read(final int[] docs, final int[] freqs) throws IOException { + final int length = docs.length; + int i = 0; + while (i < length && count < numDocs) { + final int docCode = stream.readVInt(); + doc += docCode >>> 1; // shift off low bit + count++; + if (deletedDocs == null || !deletedDocs.get(doc)) { + docs[i] = doc; + freqs[i] = 1; + ++i; + } + } + return i; + } + + public boolean next() throws IOException { + while (true) { + if (count == numDocs) + return false; + int docCode = stream.readVInt(); + doc += docCode >>> 1; // shift off low bit + count++; + if (deletedDocs == null || !deletedDocs.get(doc)) + break; + } + return true; + } + + public void close() throws IOException { + stream.close(); + if (skipListReader != null) + skipListReader.close(); + } +} Index: org/apache/lucene/index/TagBlockWriter.java =================================================================== --- org/apache/lucene/index/TagBlockWriter.java (revision 0) +++ org/apache/lucene/index/TagBlockWriter.java (revision 0) @@ -0,0 +1,69 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; + +/** + * Can be reused by calling reset. Add docs, call finish + * + */ +public class TagBlockWriter implements TagWriteable { + private RAMOutputStream indexOutput; + private RAMOutputStream skipOutput; + private TagBlockSkipListWriter skipListWriter; + private int skipInterval; + int lastDoc = 0; + int df = 0; + + public TagBlockWriter(int skipInterval, int maxSkipLevels, int docCount) { + this.indexOutput = new RAMOutputStream(); + this.skipOutput = new RAMOutputStream(); + this.skipListWriter = new TagBlockSkipListWriter(skipInterval, maxSkipLevels, docCount, indexOutput); + this.skipInterval = skipInterval; + } + + public int getDocFreq() { + return df; + } + + public void add(int doc) throws IOException { + if ((++df % skipInterval) == 0) { + skipListWriter.setSkipData(lastDoc); + skipListWriter.bufferSkip(df); + } + final int newDocCode = (doc - lastDoc) << 1; + lastDoc = doc; + indexOutput.writeVInt(newDocCode | 1); + } + + public void write(IndexOutput output) throws IOException { + skipListWriter.writeSkip(skipOutput); + output.writeVInt((int)skipOutput.length()); + indexOutput.writeTo(output); + skipOutput.writeTo(output); + } + + //public void finish() throws IOException { + // skipListWriter.writeSkip(skipOutput); + //} + + public void reset() { + indexOutput.reset(); + skipOutput.reset(); + skipListWriter.resetSkip(); + } + + public int getSize() { + return getDocsSize() + getSkipSize(); + } + + public int getDocsSize() { + return (int)indexOutput.length(); + } + + public int getSkipSize() { + return (int)skipOutput.length(); + } +} Index: org/apache/lucene/index/TagFieldData.java =================================================================== --- org/apache/lucene/index/TagFieldData.java (revision 0) +++ org/apache/lucene/index/TagFieldData.java (revision 0) @@ -0,0 +1,64 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.ByteArrayIndexInput; +import org.apache.lucene.store.ByteArrayIndexOutput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; + +/** + * Fields for single document. Contains term numbers to lookup actual terms + * from dictionary. + */ +public class TagFieldData { + public long version; + public Field[] fields; + + public TagFieldData(byte[] bytes) throws IOException { + ByteArrayIndexInput input = new ByteArrayIndexInput(bytes); + version = input.readVLong(); + int numFields = input.readInt(); + fields = new Field[numFields]; + for (int x=0; x < fields.length; x++) { + int fieldNum = input.readVInt(); + int termNum = input.readVInt(); + byte tlog = input.readByte(); + boolean tlogb = false; + if (tlog == 1) tlogb = true; + fields[x] = new Field(fieldNum, termNum, tlogb); + } + } + + public static class Field { + public final int fieldNum; + public final int termNum; + public final boolean tlog; + + public Field(int fieldNum, int termNum, boolean tlog) { + this.fieldNum = fieldNum; + this.termNum = termNum; + this.tlog = tlog; + } + + public void writeTo(IndexOutput output) throws IOException { + output.writeVInt(fieldNum); + output.writeVInt(termNum); + byte b = 0; + if (tlog) b = 1; + output.writeByte(b); + } + } + + public byte[] getBytes() throws IOException { + RAMOutputStream output = new RAMOutputStream(); + output.writeVLong(version); + output.writeVInt(fields.length); + for (int x=0; x < fields.length; x++) { + fields[x].writeTo(output); + } + int length = (int)output.length(); + ByteArrayIndexOutput byteArrayIndexOutput = new ByteArrayIndexOutput(length); + return byteArrayIndexOutput.toByteArray(); + } +} Index: org/apache/lucene/index/TagFieldInfos.java =================================================================== --- org/apache/lucene/index/TagFieldInfos.java (revision 0) +++ org/apache/lucene/index/TagFieldInfos.java (revision 0) @@ -0,0 +1,32 @@ +package org.apache.lucene.index; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Map; + +public class TagFieldInfos implements Serializable { + private Map fieldMap; + private Map numMap; + + public TagFieldInfos(Map fieldMap) { + this.fieldMap = fieldMap; + + } + + void add(String name, int num) { + fieldMap.put(name, num); + numMap.put(num, name); + } + + public int getMaxNum() { + return Collections.max(numMap.keySet()); + } + + public int fieldNumber(String name) { + return fieldMap.get(name); + } + + public String fieldName(int num) { + return numMap.get(num); + } +} Index: org/apache/lucene/index/TagIndex.java =================================================================== --- org/apache/lucene/index/TagIndex.java (revision 0) +++ org/apache/lucene/index/TagIndex.java (revision 0) @@ -0,0 +1,141 @@ +package org.apache.lucene.index; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; + +import jdbm.RecordManager; +import jdbm.RecordManagerFactory; +import jdbm.htree.HTree; + +import org.apache.lucene.store.Directory; + +public class TagIndex { + public static final int DOCS_PER_BLOCK = 25000; + public static final int LOWER_DOCS_PER_BLOCK = 20000; + public static final int HIGHER_DOCS_PER_BLOCK = 30000; + private int numDocsPerBlock; + FieldDatabase fieldDatabase; + Directory directory; + File fileDirectory; + int numBlocks; + int[] docsPerBlock; + int[] starts; + int maxDoc; + String rootSegment; + TagFieldInfos tagFieldInfos; // TODO: keep cached version here + + public TagIndex(int maxDoc, String rootSegment) throws IOException { + this.maxDoc = maxDoc; + this.rootSegment = rootSegment; + docsPerBlock = getDocBlocks(maxDoc); + int total = 0; + starts = new int[docsPerBlock.length + 1]; + for (int x = 0; x < docsPerBlock.length; x++) { + starts[x] = total; + total += starts[x]; + } + starts[docsPerBlock.length] = total; + fieldDatabase = new FieldDatabase(); + } + + public int addField(String name) throws IOException { + return fieldDatabase.addTagFieldInfo(name); + } + + public TagFieldInfos getTagFieldInfos() { + return tagFieldInfos; + } + + /** + * public static int[] getDocBlocks(int maxDoc) { if (maxDoc <= + * DOCS_PER_BLOCK) { return new int[] {maxDoc}; } int length = + * (int)Math.ceil((double)maxDoc / (double)DOCS_PER_BLOCK); + * System.out.println("length: "+length); int[] array = new int[length]; int + * num = DOCS_PER_BLOCK; for (int x=0; x < array.length; x++) { array[x] = + * num; num += array[x]; if (x == array.length - 1) { + * } } return array; } + */ + + public static int[] getDocBlocks(int maxDoc) { + double r = (double) maxDoc / (double) DOCS_PER_BLOCK; + // System.out.println("r: "+(int)r); + double value = Math.floor(r); + if ((maxDoc / value) > HIGHER_DOCS_PER_BLOCK) { + value = Math.ceil(r); + } + // System.out.println("value: "+(int)value); + double docsPerBlock = (double) maxDoc / value; + // System.out.println("docsPerBlock: "+(int)docsPerBlock); + int mod = maxDoc % (int) docsPerBlock; + // System.out.println("mod: "+mod); + int[] array = new int[(int) value]; + int num = (int) docsPerBlock; + for (int x = 0; x < array.length; x++) { + array[x] = (int) docsPerBlock; + num += array[x]; + } + if (mod > 0) { + array[array.length - 1] += mod; + } + // for (int x=0; x < array.length; x++) { + // System.out.println(array[x]); + // } + return array; + } + + public int getNumDocsForBlock(int i) { + return docsPerBlock[i]; + } + + /** + * Uses JDBM hash database. + * + */ + public class FieldDatabase { + public static final String TAG_FIELD_INFOS_KEY = "tagfieldinfoskey"; + private RecordManager recordManager; + HTree hashtable; + Object addTagFieldInfoLock = new Object(); + + public FieldDatabase() throws IOException { + Properties props = new Properties(); + File file = new File(fileDirectory, rootSegment + ".fdt"); + recordManager = RecordManagerFactory.createRecordManager(file.getAbsolutePath(), props); + long recid = recordManager.getNamedObject("fieldata"); + if (recid != 0) { + hashtable = HTree.load(recordManager, recid); + } else { + hashtable = HTree.createInstance(recordManager); + recordManager.setNamedObject("fieldata", hashtable.getRecid()); + } + } + + private int addTagFieldInfo(String fieldName) throws IOException { + synchronized (addTagFieldInfoLock) { + TagFieldInfos tagFieldInfos = getTagFieldInfos(); + int maxNum = tagFieldInfos.getMaxNum(); + int num = maxNum + 1; + tagFieldInfos.add(fieldName, num); + hashtable.put(TAG_FIELD_INFOS_KEY, tagFieldInfos); + return num; + } + } + + private TagFieldInfos getTagFieldInfos() throws IOException { + return (TagFieldInfos) hashtable.get(TAG_FIELD_INFOS_KEY); + } + + public TagFieldData get(int doc) throws IOException { + byte[] bytes = (byte[]) hashtable.get(new Integer(doc)); + return new TagFieldData(bytes); + } + + public void update(int doc, TagFieldData fieldData) throws IOException { + // TODO: merge old and new field data + byte[] oldBytes = (byte[]) hashtable.get(new Integer(doc)); + TagFieldData oldData = new TagFieldData(oldBytes); + hashtable.put(new Integer(doc), fieldData.getBytes()); + } + } +} Index: org/apache/lucene/index/TagIndexSnapshot.java =================================================================== --- org/apache/lucene/index/TagIndexSnapshot.java (revision 0) +++ org/apache/lucene/index/TagIndexSnapshot.java (revision 0) @@ -0,0 +1,663 @@ +package org.apache.lucene.index; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.TagIndexSnapshot.Log.TransactionLogTermEnum; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.TagRAMOutputStream; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.LimitedIndexInput; +import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.PriorityQueue; + +// TODO: add lrucache for blocks, tagblockinfo, tagblockinfos +public class TagIndexSnapshot { + private Log log; + private Index index; + private BitVector deletedDocs; + TagIndex tagIndex; + private long version; + private String segment; + private ReentrantLock newTagIndexSnapshotLock = new ReentrantLock(); + + public TagIndexSnapshot(String segment, long version, Log log, BitVector deletedDocs) throws IOException { + this.segment = segment; + this.version = version; + this.deletedDocs = deletedDocs; + this.log = log; + } + + public TagBlockInfos getTagBlockInfos(Term term) throws IOException { + TagTermInfo tagTermInfo = getTagTermInfo(term); + return getTagBlockInfos(tagTermInfo); + } + + public TagBlockInfos getTagBlockInfos(TagTermInfo tagTermInfo) throws IOException { + if (tagTermInfo instanceof TermDiskInfo) { + return log.getTagBlockInfos((TermDiskInfo)tagTermInfo); + } else { + return index.getTagBlockInfos(tagTermInfo); + } + } + + public TagTermInfo getTagTermInfo(Term term) throws IOException { + TagTermInfo tagTermInfo = log.getTagTermInfo(term); + if (tagTermInfo != null) return tagTermInfo; + return index.getTagTermInfo(term); + } + + public TermDocs termDocs() { + return new TagTermDocs(this, tagIndex.starts, deletedDocs, index.getInput(), log.input, index.termInfosReader.getSkipInterval(), index.termInfosReader + .getMaxSkipLevels()); + } + + public TagMultiTermEnum terms(Term term) throws IOException { + TagSegmentTermEnum tagSegmentTermEnum = index.terms(term); + TransactionLogTermEnum logTermEnum = log.terms(term); + return new TagMultiTermEnum(tagSegmentTermEnum, logTermEnum, term, log.deletedIndexTerms); + } + + /** + * Copy this snapshot increment version + */ + public TagIndexSnapshot newTagIndexSnapshot(String segment, BitVector deleted) throws IOException { + newTagIndexSnapshotLock.lock(); + try { + long newVersion = version + 1; + if (deleted == null) { + deleted = deletedDocs; + } + Log newLog = (Log) log.copy(); + return new TagIndexSnapshot(segment, newVersion, newLog, deleted); + } finally { + newTagIndexSnapshotLock.unlock(); + } + } + + public long getVersion() { + return version; + } + + // TODO: store term as bytes with sorted field num as first byte + public static class TermDiskInfo extends TagTermInfo implements TagWriteable { + public Term term; + + public TermDiskInfo() {} + + public TermDiskInfo(IndexInput input) throws IOException { + docFreq = input.readVInt(); + termNum = input.readVInt(); + blockInfosPointer = input.readVLong(); + String field = input.readString(); + String text = input.readString(); + term = new Term(field, text); + } + + public void write(IndexOutput output) throws IOException { + output.writeVInt(docFreq); + output.writeVInt(termNum); + output.writeVLong(blockInfosPointer); + output.writeString(term.field()); + output.writeString(term.text()); + } + } + + public static class Record { + public int type; + public IndexInput input; + public int length; + + public Record(int type, int length, IndexInput input) { + this.type = type; + this.length = length; + this.input = input; + } + + public void skip() throws IOException { + input.seek(input.getFilePointer() + length); + } + } + + /** + * Write record methods return pointer to the actual data + * + */ + // TODO: add lrucache of tagblockinfos and tagblocks + public class Log { + public static final long CURRENT_FORMAT = 1; + public static final int TERM_DISK_INFO = 4; + public static final int BLOCK_DATA = 10; + public static final int BLOCK_INFOS = 7; + private ArrayList termDiskInfos = new ArrayList(); // sorted + // by + // term + private ArrayList numTermDiskInfos = new ArrayList(); // sorted + // by + // termnum. + // for + // log + // termnum is an id + private HashSet deletedIndexTerms; // terms from index that are not + private IndexInput input; + private IndexOutput output; + int skipInterval; + int maxSkipLevels; + int termIdSequence = 0; + private TermNumDiskComparator termNumDiskComparator = new TermNumDiskComparator(); + private TermDiskComparator termDiskComparator = new TermDiskComparator(); + private ReentrantLock addTermDiskInfoLock = new ReentrantLock(); + private String fileName; + + public Log(int skipInterval, int maxSkipLevels) throws IOException { + this.skipInterval = skipInterval; + this.maxSkipLevels = maxSkipLevels; + fileName = segment + "." + version + ".tlg"; + output = tagIndex.directory.createOutput(fileName); + input = tagIndex.directory.openInput(fileName); + if (output.length() > 0) { + load(); + } else { + // TODO: write out headers + output.writeLong(CURRENT_FORMAT); + } + } + + public TagBlockInfos getTagBlockInfos(TermDiskInfo tdi) throws IOException { + IndexInput i = (IndexInput)input.clone(); + try { + return new TagBlockInfos(tdi.termNum, true, tdi.blockInfosPointer, i, 0); + } finally { + i.close(); + } + } + + public TagTermInfo getTagTermInfo(Term term) { + TransactionLogTermEnum termEnum = terms(term); + TagTermInfo tagTermInfo = termEnum.termDiskInfo(); + return tagTermInfo; + } + + public Log copy() { + Log log = new Log(); + log.termDiskInfos = (ArrayList) termDiskInfos.clone(); + log.numTermDiskInfos = (ArrayList) numTermDiskInfos.clone(); + log.deletedIndexTerms = (HashSet) deletedIndexTerms.clone(); + log.skipInterval = skipInterval; + log.maxSkipLevels = maxSkipLevels; + + return log; + } + + public synchronized TermDiskInfo addTermDiskInfo(Term term, long blockInfosPointer, int docFreq) throws IOException { + TermDiskInfo key = new TermDiskInfo(); + key.term = term; + int index = Collections.binarySearch(termDiskInfos, key, termNumDiskComparator); + if (index < 0) + index = -1 - index; + TermDiskInfo termDiskInfo = new TermDiskInfo(); + termDiskInfo.blockInfosPointer = blockInfosPointer; + termDiskInfo.docFreq = docFreq; + termDiskInfo.term = term; + termDiskInfo.termNum = termIdSequence; + termDiskInfos.add(index, termDiskInfo); + int numIndex = Collections.binarySearch(numTermDiskInfos, key, termNumDiskComparator); + if (numIndex < 0) + numIndex = -1 - numIndex; + numTermDiskInfos.add(numIndex, termDiskInfo); + termIdSequence++; + writeRecord(termDiskInfo); + return termDiskInfo; + } + + public void close() throws IOException { + if (output != null) + output.close(); + } + + public long writeRecord(int type, TagWriteable writeable) throws IOException { + TagRAMOutputStream byteOutput = new TagRAMOutputStream(); + writeable.write(byteOutput); + return writeRecord(type, byteOutput); + } + + public long writeRecord(int type, TagRAMOutputStream byteOutput) throws IOException { + output.writeVInt(type); + int length = (int) output.length(); + byteOutput.writeVInt(length); + long pointer = output.getFilePointer(); + byteOutput.writeTo(output); + return pointer; + } + + public long writeRecord(byte type, byte[] bytes) throws IOException { + output.writeVInt(type); + long pointer = output.getFilePointer(); + output.writeVInt(bytes.length); + output.writeBytes(bytes, bytes.length); + return pointer; + } + + public Record readRecord() throws IOException { + int type = input.readVInt(); + int length = input.readVInt(); + LimitedIndexInput limitedIndexInput = new LimitedIndexInput(length, input); + return new Record(type, length, limitedIndexInput); + } + + public long writeRecord(TermDiskInfo termDiskInfo) throws IOException { + return writeRecord(TERM_DISK_INFO, termDiskInfo); + } + + public long writeRecord(TagBlockInfos blockInfos) throws IOException { + return writeRecord(BLOCK_INFOS, blockInfos); + } + /** + public long writeRecord(TagBlockInfo tagBlockInfo) throws IOException { + return writeRecord(BLOCK_INFO, tagBlockInfo); + } + + public long writeRecord(TagBlockInfo[] blockInfoArray) throws IOException { + ByteArrayIndexOutput2 byteOutput = new ByteArrayIndexOutput2(); + byteOutput.writeVInt(blockInfoArray.length); + for (int x=0; x < blockInfoArray.length; x++) { + blockInfoArray[x].write(byteOutput); + } + return writeRecord(BLOCK_INFOS_ARRAY, byteOutput); + } + **/ + public long writeRecord(TagBlockWriter tagBlockWriter) throws IOException { + return writeRecord(BLOCK_DATA, tagBlockWriter); + } + + public void load() throws IOException { + Map termInfoMap = new HashMap(2000); + long formatVersion = input.readLong(); + while (true) { + try { + Record record = readRecord(); + if (record.type == TERM_DISK_INFO) { + TermDiskInfo tdi = new TermDiskInfo(input); + termInfoMap.put(tdi.termNum, tdi); + } else { + record.skip(); + } + } catch (EOFException eofException) { + break; + } + } + termDiskInfos.ensureCapacity(termInfoMap.size()); + numTermDiskInfos.ensureCapacity(termInfoMap.size()); + for (TermDiskInfo termDiskInfo : termInfoMap.values()) { + termDiskInfos.add(termDiskInfo); + numTermDiskInfos.add(termDiskInfo); + } + Collections.sort(termDiskInfos, termDiskComparator); + Collections.sort(numTermDiskInfos, termNumDiskComparator); + } + + public synchronized void update(Term term, DocIdSet updates, boolean add) throws IOException { + IndexInput logInput = (IndexInput) input.clone(); + IndexInput indexInput = null; + int numBlocks = tagIndex.numBlocks; + try { + int[] starts = tagIndex.starts; + OpenBitSet[] updateBitSets = getAffectedBlocks(updates, starts); + TagBlockInfos tbi = TagIndexSnapshot.this.getTagBlockInfos(term); + TagBlockInfos newTbi = new TagBlockInfos(); + newTbi.termNum = tbi.termNum; // TODO: need to copy arrays + newTbi.positions = tbi.positions; + newTbi.docFreqs = tbi.docFreqs; + newTbi.tlog = true; + newTbi.tlogs = tbi.tlogs; + int docFreq = 0; + TagBlockMerger[] tagBlockMergers = new TagBlockMerger[numBlocks]; + for (int x = 0; x < numBlocks; x++) { + if (updateBitSets[x] != null) { + IndexInput input = indexInput; + if (tbi.tlogs[x]) + input = logInput; + TagBlockTermDocs tagBlockTermDocs = new TagBlockTermDocs(x, tbi, deletedDocs, input, skipInterval, maxSkipLevels); + tagBlockMergers[x] = new TagBlockMerger(tagBlockTermDocs, tbi.docFreqs[x], skipInterval, maxSkipLevels); + int blockDocFreq = tagBlockMergers[x].merge(add, updateBitSets[x]); + docFreq += blockDocFreq; + } + } + for (int x=0; x < numBlocks; x++) { + if (tagBlockMergers[x] != null) { + long position = writeRecord(tagBlockMergers[x].tagBlockData); // write blockdata + newTbi.positions[x] = position; + newTbi.docFreqs[x] = tagBlockMergers[x].docFreq; + newTbi.tlogs[x] = true; + } + } + long blockInfosPointer = writeRecord(newTbi); // write tagblockinfos + TermDiskInfo termDiskInfo = addTermDiskInfo(term, blockInfosPointer, docFreq); // write terminfo + } finally { + logInput.close(); + } + } + + public OpenBitSet[] getAffectedBlocks(DocIdSet updates, int[] starts) throws IOException { + DocIdSetIterator iterator = updates.iterator(); + int pos = 0; + OpenBitSet[] blocks = new OpenBitSet[starts.length - 1]; + while (iterator.next()) { + int doc = iterator.doc(); + while (pos < blocks.length) { + if (starts[pos] <= doc && doc < starts[pos + 1]) { + if (blocks[pos] == null) + blocks[pos] = new OpenBitSet(tagIndex.getNumDocsForBlock(pos)); + blocks[pos].set(doc - starts[pos]); + break; + } else { + pos++; + } + } + } + return blocks; + } + + public Term getTermByNum(int termNum) { + TermDiskInfo key = new TermDiskInfo(); + key.termNum = termNum; + int index = Collections.binarySearch(numTermDiskInfos, key, termNumDiskComparator); + if (index < 0) + return null; + return ((TermDiskInfo) numTermDiskInfos.get(index)).term; + } + + public boolean isIndexTermDeleted(int num) { + return deletedIndexTerms.contains(num); + } + + public TransactionLogTermDocs termDocs() { + return new TransactionLogTermDocs(deletedDocs, skipInterval, maxSkipLevels); + } + /** + private TagBlockInfo[] loadBlockInfos(long blockInfosPointer) throws IOException { + TagBlockInfos tagBlockInfos = new TagBlockInfos(true, blockInfosPointer, input); + TagBlockInfo[] blockInfos = TagBlockInfos.load(true, tagBlockInfos, input); + return blockInfos; + } + **/ + public class TransactionLogTermDocs extends TagBaseBlocksTermDocs { + IndexInput input; + + public TransactionLogTermDocs(BitVector deletedDocs, int skipInterval, int maxSkipLevels) { + super(TagIndexSnapshot.this, deletedDocs, skipInterval, maxSkipLevels); + } + + public void seek(Term term) throws IOException { + TransactionLogTermEnum termEnum = terms(term); + seek(termEnum); + } + + public void seek(TermEnum termEnum) throws IOException { + TransactionLogTermEnum transactionLogTermEnum = (TransactionLogTermEnum) termEnum; + TermDiskInfo termDiskInfo = transactionLogTermEnum.termDiskInfo(); + TagBlockInfos tagBlockInfos = new TagBlockInfos(termDiskInfo.termNum, true, termDiskInfo.blockInfosPointer, input, 0); + seek(tagBlockInfos, input, term); + } + } + + public TransactionLogTermEnum terms(Term term) { + return new TransactionLogTermEnum(term); + } + + public class TransactionLogTermEnum extends TagTermEnum { + int pos; + + private TransactionLogTermEnum(Term fromTerm) { + TermDiskInfo from = new TermDiskInfo(); + from.term = fromTerm; + pos = Collections.binarySearch(termDiskInfos, from, new TermDiskComparator()); + if (pos < 0) { + pos = -1 - pos; + } + } + + public boolean next() throws IOException { + if (pos < termDiskInfos.size() - 1) { + pos++; + return true; + } + return false; + } + + public int termNum() { + return termDiskInfo().termNum; + } + + public TagTermInfo termInfo() { + return termDiskInfo(); + } + + public TermDiskInfo termDiskInfo() { + return (TermDiskInfo) termDiskInfos.get(pos); + } + + public Term term() { + return termDiskInfo().term; + } + + public int docFreq() { + return 1; + } + + public void close() throws IOException { + } + } + } + + public static class TermDiskComparator implements Comparator { + public int compare(Object o1, Object o2) { + TermDiskInfo t1 = (TermDiskInfo) o1; + TermDiskInfo t2 = (TermDiskInfo) o2; + return t1.term.compareTo(t2.term); + } + } + + public static class TermNumDiskComparator implements Comparator { + public int compare(Object o1, Object o2) { + TermDiskInfo t1 = (TermDiskInfo) o1; + TermDiskInfo t2 = (TermDiskInfo) o2; + if (t1.termNum > t2.termNum) + return 1; + else if (t1.termNum < t2.termNum) + return -1; + return 0; + } + } + + public class Index { + private TagTermInfosReader termInfosReader; + + public Index(TagFieldInfos tagFieldInfos) throws IOException { + termInfosReader = new TagTermInfosReader(tagIndex.directory, segment, tagFieldInfos); + } + + public TagBlockInfos getTagBlockInfos(TagTermInfo tti) throws IOException { + IndexInput i = (IndexInput)termInfosReader.getInput().clone(); + try { + return new TagBlockInfos(tti.termNum, false, tti.blockInfosPointer, i, tti.basePointer); + } finally { + i.close(); + } + } + + public TagTermInfo getTagTermInfo(Term term) throws IOException { + TagSegmentTermEnum termEnum = terms(term); + return termEnum.termInfo(); + } + + public IndexInput getInput() { + return termInfosReader.getInput(); + } + + public Term getTermByNum(int termNum) throws IOException { + return termInfosReader.getByNum(termNum).term; + } + + public TagSegmentTermEnum terms(Term term) throws IOException { + return termInfosReader.terms(term); + } + } + + public class TagMultiTermEnum extends TagTermEnum { + private TermEnumMergeQueue queue; + private TagTermInfo termInfo; + private Term term; + private int docFreq = 1; + private Set indexDeletedTermNums; + + public TagMultiTermEnum(TagSegmentTermEnum tagSegmentTermEnum, TransactionLogTermEnum transactionLogTermEnum, Term t, + Set indexDeletedTermNums) throws IOException { + this.indexDeletedTermNums = indexDeletedTermNums; + queue = new TermEnumMergeQueue(2); + initTermEnum(t, tagSegmentTermEnum); + initTermEnum(t, transactionLogTermEnum); + if (t != null && queue.size() > 0) { + next(); + } + } + + public int termNum() { + return termInfo.termNum; + } + + TagTermInfo termInfo() { + return termInfo; + } + + private void initTermEnum(Term t, TermEnum termEnum) throws IOException { + if (t == null ? termEnum.next() : termEnum.term() != null) + queue.put(termEnum); // initialize queue + else + termEnum.close(); + } + + public boolean next() throws IOException { + TermEnum top = (TermEnum) queue.top(); + if (top == null) { + term = null; + return false; + } + term = top.term(); + TagTermInfo logTagTermInfo = null; + TagTermInfo indexTagTermInfo = null; + while (top != null && term.compareTo(top.term()) == 0) { + queue.pop(); + if (top instanceof TagSegmentTermEnum) { + logTagTermInfo = ((TagSegmentTermEnum) top).termInfo(); + } else { + indexTagTermInfo = ((TransactionLogTermEnum) top).termDiskInfo(); + } + if (top.next()) { + queue.put(top); + } else + top.close(); + top = (TermEnum) queue.top(); + } + if (logTagTermInfo != null) + termInfo = logTagTermInfo; + else + termInfo = indexTagTermInfo; + // skip deleted terms + if (termInfo == indexTagTermInfo && indexDeletedTermNums.contains(termInfo.termNum)) { + return next(); + } + return true; + } + + public Term term() { + return term; + } + + public int docFreq() { + return docFreq; + } + + public void close() throws IOException { + queue.close(); + } + } + + public OpenBitSet[] separateDocs(OpenBitSet changes, int[] starts) { + int numBlocks = starts.length - 1; + OpenBitSet[] bitSets = new OpenBitSet[numBlocks]; + for (int x = 0; x < starts.length - 1; x++) { + int blockLength = tagIndex.getNumDocsForBlock(x); + bitSets[x] = docs(changes, starts[x], starts[x + 1] - 1, blockLength); + } + return bitSets; + } + + public static OpenBitSet docs(OpenBitSet changes, int start, int end, int blockLength) { + OpenBitSet bitSet = null; + int pos = 0; + for (int x = start; x <= end; x++) { + if (changes.get(x)) { + if (bitSet == null) + bitSet = new OpenBitSet(blockLength); + bitSet.set(pos); + } + } + return bitSet; + } + + public Document document(int n) throws CorruptIndexException, IOException { + TagFieldData fieldsData = tagIndex.fieldDatabase.get(n); + Document document = new Document(); + for (int x = 0; x < fieldsData.fields.length; x++) { + String name = tagIndex.getTagFieldInfos().fieldName(fieldsData.fields[x].fieldNum); + Term term = null; + if (fieldsData.fields[x].tlog) { + term = log.getTermByNum(fieldsData.fields[x].termNum); + } else { + term = index.getTermByNum(fieldsData.fields[x].termNum); + } + assert name.equals(term.field()); + document.add(new Field(name, term.text(), Field.Store.YES, Field.Index.UN_TOKENIZED)); + } + return document; + } + + final static class TermEnumMergeQueue extends PriorityQueue { + TermEnumMergeQueue(int size) { + initialize(size); + } + + protected final boolean lessThan(Object a, Object b) { + TermEnum stiA = (TermEnum) a; + TermEnum stiB = (TermEnum) b; + int comparison = stiA.term().compareTo(stiB.term()); + if (comparison == 0) { + if (stiA instanceof TagSegmentTermEnum) + return true; + else + return false; + } else + return comparison < 0; + } + + final void close() throws IOException { + while (top() != null) + ((TermEnum) pop()).close(); + } + } +} Index: org/apache/lucene/index/TagIndexWriter.java =================================================================== --- org/apache/lucene/index/TagIndexWriter.java (revision 0) +++ org/apache/lucene/index/TagIndexWriter.java (revision 0) @@ -0,0 +1,79 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.store.Directory; + +public class TagIndexWriter { + private Map postings = new HashMap(); + Directory directory; + String segment; + TagFieldInfos fis; + int indexInterval; + + public TagIndexWriter(Directory directory, String segment, TagFieldInfos fis, int indexInterval) { + this.directory = directory; + this.segment = segment; + this.fis = fis; + this.indexInterval = indexInterval; + } + + public void add(Term term, int doc) { + Posting posting = postings.get(term); + if (posting == null) { + posting = new Posting(term); + postings.put(term, posting); + } + posting.docs.add(doc); + } + + public static class Posting implements Comparable { + private Term term; + private List docs = new ArrayList(); + + public Posting(Term term) { + this.term = term; + } + + public int compareTo(Object object) { + Posting other = (Posting)object; + return term.compareTo(other.term); + } + } + + public void close() throws IOException { + List sorted = new ArrayList(postings.values()); + Collections.sort(sorted); + TagTermInfosWriter tagTermInfosWriter = new TagTermInfosWriter(directory, segment, fis, indexInterval); + for (int x=0; x < sorted.size(); x++) { + Posting posting = sorted.get(x); + TagTermInfo ti = new TagTermInfo(); + ti.docFreq = posting.docs.size(); + ti.termNum = x; + int[] array = new int[posting.docs.size()]; + for (int i=0; i < array.length; i++) { + array[i] = posting.docs.get(i); + } + Arrays.sort(array); // should already be sorted + tagTermInfosWriter.add(posting.term, x, array); + } + tagTermInfosWriter.close(); + } + + public void addDocument(int docNum, Document document) { + List fields = document.getFields(); + for (int x=0; x < fields.size(); x++) { + Field field = (Field)fields.get(x); + Term term = new Term(field.name(), field.stringValue()); + add(term, docNum); + } + } +} Index: org/apache/lucene/index/TagSegmentTermEnum.java =================================================================== --- org/apache/lucene/index/TagSegmentTermEnum.java (revision 0) +++ org/apache/lucene/index/TagSegmentTermEnum.java (revision 0) @@ -0,0 +1,179 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; + +public class TagSegmentTermEnum extends TagTermEnum { + IndexInput input; + TagFieldInfos tagFieldInfos; + long size; + long position = -1; + + private TagTermBuffer termBuffer = new TagTermBuffer(); + private TagTermBuffer prevBuffer = new TagTermBuffer(); + private TagTermBuffer scanBuffer = new TagTermBuffer(); // used for scanning + + private TagTermInfo termInfo = new TagTermInfo(); + + private int format; + private boolean isIndex = false; + long indexPointer = 0; + int indexInterval; + int skipInterval; + int maxSkipLevels; + int prevNum; + int scanNum; + + TagSegmentTermEnum(IndexInput i, TagFieldInfos tagFieldInfos, boolean isi) throws CorruptIndexException, IOException { + input = i; + this.tagFieldInfos = tagFieldInfos; + isIndex = isi; + maxSkipLevels = 1; // use single-level skip lists for formats > -3 + + int firstInt = input.readInt(); + format = firstInt; + + // check that it is a format we can understand + if (format < TermInfosWriter.FORMAT_CURRENT) + throw new CorruptIndexException("Unknown format version:" + format + " expected " + TermInfosWriter.FORMAT_CURRENT + " or higher"); + + size = input.readLong(); // read the size + + indexInterval = input.readInt(); + skipInterval = input.readInt(); + maxSkipLevels = input.readInt(); + } + + protected Object clone() { + TagSegmentTermEnum clone = null; + try { + clone = (TagSegmentTermEnum) super.clone(); + } catch (CloneNotSupportedException e) { + } + + clone.input = (IndexInput) input.clone(); + clone.termInfo = new TagTermInfo(termInfo); + + clone.termBuffer = (TagTermBuffer) termBuffer.clone(); + clone.prevBuffer = (TagTermBuffer) prevBuffer.clone(); + clone.scanBuffer = new TagTermBuffer(); + + return clone; + } + + final void seek(long pointer, int p, Term t, TagTermInfo ti) throws IOException { + input.seek(pointer); + position = p; + termBuffer.set(t); + prevBuffer.reset(); + termInfo.set(ti); + } + + /** Increments the enumeration to the next element. True if one exists. */ + public boolean next() throws IOException { + if (position++ >= size - 1) { + prevBuffer.set(termBuffer); + prevNum = termInfo.termNum; + termBuffer.reset(); + return false; + } + prevBuffer.set(termBuffer); + prevNum = termInfo.termNum; + + termBuffer.read(input, tagFieldInfos); + termInfo.docFreq = input.readVInt(); + termInfo.termNum = input.readVInt(); + if (!isIndex) { + long basePointer = input.getFilePointer(); + int blocksLength = input.readVInt(); + int infosLength = input.readVInt(); + termInfo.blockInfosPointer = basePointer + blocksLength; + termInfo.basePointer = basePointer; + input.seek(basePointer + blocksLength + infosLength); + } else { + indexPointer += input.readVLong(); + } + return true; + } + + final int scanTo(int termNum) throws IOException { + scanNum = termNum; + int count = 0; + while (scanNum > termNum && next()) { + count++; + } + return count; + } + + /** + * Optimized scan, without allocating new terms. Return number of invocations + * to next(). + */ + final int scanTo(Term term) throws IOException { + scanBuffer.set(term); + int count = 0; + while (scanBuffer.compareTo(termBuffer) > 0 && next()) { + count++; + } + return count; + } + + /** + * Returns the current Term in the enumeration. Initially invalid, valid after + * next() called for the first time. + */ + public final Term term() { + return termBuffer.toTerm(); + } + + public final int termNum() { + return termInfo.termNum; + } + + final int prevNum() { + return prevNum; + } + + /** Returns the previous Term enumerated. Initially null. */ + final Term prev() { + return prevBuffer.toTerm(); + } + + /** + * Returns the current TermInfo in the enumeration. Initially invalid, valid + * after next() called for the first time. + */ + final TagTermInfo termInfo() { + return new TagTermInfo(termInfo); + } + + /** + * Sets the argument to the current TermInfo in the enumeration. Initially + * invalid, valid after next() called for the first time. + */ + final void termInfo(TagTermInfo ti) { + ti.set(termInfo); + } + + /** + * Returns the docFreq from the current TermInfo in the enumeration. Initially + * invalid, valid after next() called for the first time. + */ + public final int docFreq() { + return termInfo.docFreq; + } + + /* + * Returns the blockInfosPointer from the current TermInfo in the enumeration. + * Initially invalid, valid after next() called for the first time. + */ + final long blockInfosPointer() { + return termInfo.blockInfosPointer; + } + + /** Closes the enumeration to further activity, freeing resources. */ + public final void close() throws IOException { + input.close(); + } +} Index: org/apache/lucene/index/TagTermBuffer.java =================================================================== --- org/apache/lucene/index/TagTermBuffer.java (revision 0) +++ org/apache/lucene/index/TagTermBuffer.java (revision 0) @@ -0,0 +1,109 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.UnicodeUtil; + +public class TagTermBuffer implements Cloneable { + private String field; + private Term term; // cached + private boolean dirty; // true if text was set + // externally (ie not read + // via UTF8 bytes) + + private UnicodeUtil.UTF16Result text = new UnicodeUtil.UTF16Result(); + private UnicodeUtil.UTF8Result bytes = new UnicodeUtil.UTF8Result(); + + public final int compareTo(TagTermBuffer other) { + if (field == other.field) // fields are interned + return compareChars(text.result, text.length, other.text.result, other.text.length); + else + return field.compareTo(other.field); + } + + private static final int compareChars(char[] chars1, int len1, char[] chars2, int len2) { + final int end = len1 < len2 ? len1 : len2; + for (int k = 0; k < end; k++) { + char c1 = chars1[k]; + char c2 = chars2[k]; + if (c1 != c2) { + return c1 - c2; + } + } + return len1 - len2; + } + + public final void read(IndexInput input, TagFieldInfos fieldInfos) throws IOException { + this.term = null; // invalidate cache + int start = input.readVInt(); + int length = input.readVInt(); + int totalLength = start + length; + if (dirty) { + // Fully convert all bytes since bytes is dirty + UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes); + bytes.setLength(totalLength); + input.readBytes(bytes.result, start, length); + UnicodeUtil.UTF8toUTF16(bytes.result, 0, totalLength, text); + dirty = false; + } else { + // Incrementally convert only the UTF8 bytes that are new: + bytes.setLength(totalLength); + input.readBytes(bytes.result, start, length); + UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text); + } + this.field = fieldInfos.fieldName(input.readVInt()); + } + + public final void set(Term term) { + if (term == null) { + reset(); + return; + } + final String termText = term.text(); + final int termLen = termText.length(); + text.setLength(termLen); + termText.getChars(0, termLen, text.result, 0); + dirty = true; + field = term.field(); + this.term = term; + } + + public final void set(TagTermBuffer other) { + text.copyText(other.text); + dirty = true; + field = other.field; + term = other.term; + } + + public void reset() { + field = null; + text.setLength(0); + term = null; + dirty = true; + } + + public Term toTerm() { + if (field == null) // unset + return null; + + if (term == null) + term = new Term(field, new String(text.result, 0, text.length), false); + + return term; + } + + protected Object clone() { + TagTermBuffer clone = null; + try { + clone = (TagTermBuffer) super.clone(); + } catch (CloneNotSupportedException e) { + } + + clone.dirty = true; + clone.bytes = new UnicodeUtil.UTF8Result(); + clone.text = new UnicodeUtil.UTF16Result(); + clone.text.copyText(text); + return clone; + } +} \ No newline at end of file Index: org/apache/lucene/index/TagTermDocs.java =================================================================== --- org/apache/lucene/index/TagTermDocs.java (revision 0) +++ org/apache/lucene/index/TagTermDocs.java (revision 0) @@ -0,0 +1,243 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.index.TagIndexSnapshot.TagMultiTermEnum; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitVector; + +public class TagTermDocs implements TermDocs { + private TagIndexSnapshot snapshot; + private TagBlockTermDocs current; + private TagBlockTermDocs[] termDocsArray; + private TagBlockInfos blockInfos; + private final int[] starts; + int pointer; + int base; + private final BitVector deletedDocs; + IndexInput indexInput; + IndexInput tlogInput; + int skipInterval; + int maxSkipLevels; + Term term; + + public TagTermDocs(TagIndexSnapshot snapshot, int[] starts, BitVector deletedDocs, IndexInput indexInput, IndexInput tlogInput, int skipInterval, int maxSkipLevels) { + this.snapshot = snapshot; + this.starts = starts; + this.deletedDocs = deletedDocs; + this.indexInput = (IndexInput) indexInput.clone(); + this.tlogInput = (IndexInput) tlogInput.clone(); + this.skipInterval = skipInterval; + this.maxSkipLevels = maxSkipLevels; + } + + public void seek(TermEnum termEnum) throws IOException { + TagMultiTermEnum tagMultiTermEnum = (TagMultiTermEnum) termEnum; + TagTermInfo tagTermInfo = tagMultiTermEnum.termInfo(); + seek(tagTermInfo, termEnum.term()); + } + + public void seek(Term term) throws IOException { + TagTermInfo tti = snapshot.getTagTermInfo(term); + seek(tti, term); + } + + public int freq() { + return 1; + } + + public void close() throws IOException { + indexInput.close(); + tlogInput.close(); + } + + void seek(TagTermInfo ti, Term term) throws IOException { + this.term = term; + this.base = 0; + this.pointer = 0; + this.current = null; + // load blockinfos + blockInfos = snapshot.getTagBlockInfos(term); + /** + TagBlockInfo[] indexBlockInfos = loadIndexBlockInfos(ti.blockInfosPointer); + Set blockNumSet = new HashSet(tlogBlockInfos.length + indexBlockInfos.length); + for (int x=0; x < tlogBlockInfos.length; x++) { + TagBlockInfo tagBlockInfo = tlogBlockInfos[x]; + blockNumSet.add(tagBlockInfo.blockNum); + } + List goodIndexBlockInfos = new ArrayList(indexBlockInfos.length + tlogBlockInfos.length); + for (int x=0; x < indexBlockInfos.length; x++) { + TagBlockInfo tagBlockInfo = indexBlockInfos[x]; + if (!blockNumSet.contains(tagBlockInfo.blockNum)) blockNumSet.add(tagBlockInfo.blockNum); + } + blockInfos = new TagBlockInfo[goodIndexBlockInfos.size()]; + for (int x=0; x < blockInfos.length; x++) { + blockInfos[x] = (TagBlockInfo)goodIndexBlockInfos.get(x); + } + blockInfos = null; + **/ + } + /** + private TagBlockInfo[] loadIndexBlockInfos(long blockInfosPointer) throws IOException { + indexInput.seek(blockInfosPointer); + int num = indexInput.readVInt(); + TagBlockInfo[] array = new TagBlockInfo[num]; + for (int x = 0; x < num; x++) { + TagBlockInfo tagBlockInfo = new TagBlockInfo(); + // TODO: should be able to calculate docstart and maxdocs from blocknum + tagBlockInfo.docStart = indexInput.readVInt(); + tagBlockInfo.filePointer = indexInput.readVLong(); + tagBlockInfo.skipOffset = indexInput.readVInt(); + tagBlockInfo.numDocs = indexInput.readVInt(); + tagBlockInfo.blockNum = indexInput.readVInt(); + tagBlockInfo.maxDoc = indexInput.readVInt(); + tagBlockInfo.tlog = false; + array[x] = tagBlockInfo; + } + return array; + } + **/ + public int doc() { + return base + current.doc(); + } + + public boolean next() throws IOException { + for (;;) { + if (current != null && current.next()) { + return true; + } else if (pointer < termDocsArray.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return false; + } + } + } + + private TagBlockTermDocs termDocs(int i) throws IOException { + if (termDocsArray[i] == null) { + IndexInput stream = null; + if (blockInfos.tlogs[i]) { + stream = tlogInput; + } else { + stream = indexInput; + } + termDocsArray[i] = new TagBlockTermDocs(i, blockInfos, deletedDocs, stream, skipInterval, maxSkipLevels); + } + return termDocsArray[i]; + } + + public int read(final int[] docs, final int[] freqs) throws IOException { + while (true) { + while (current == null) { + if (pointer < termDocsArray.length) { + base = starts[pointer]; + current = termDocs(pointer++); + } else { + return 0; + } + } + int end = current.read(docs, freqs); + if (end == 0) { + current = null; + } else { + final int b = base; + for (int i = 0; i < end; i++) + docs[i] += b; + return end; + } + } + } + + public boolean skipTo(int target) throws IOException { + for (;;) { + if (current != null && current.skipTo(target - base)) { + return true; + } else if (pointer < termDocsArray.length) { + //pointer = blockIndex(target, starts, blockInfos.length, pointer); + pointer = binarySearch(starts, target, pointer, termDocsArray.length - pointer); + base = starts[pointer]; + current = termDocs(pointer); + } else + return false; + } + } + + // from instantiatedindex + public static final int binarySearch(int[] array, int key, int offset, int length) {//min, int max) { + if (length == 0) { + return -1 - offset; + } + int min = offset, max = offset + length - 1; + int minVal = array[min], maxVal = array[max]; + + int nPreviousSteps = 0; + + // Uncomment these two lines to get statistics about the average number of steps in the test report : + //totalCalls++; + for (;;) { + //totalSteps++; + + // be careful not to compute key - minVal, for there might be an integer overflow. + if (key <= minVal) return key == minVal ? min : -1 - min; + if (key >= maxVal) return key == maxVal ? max : -2 - max; + + assert min != max; + + int pivot; + // A typical binarySearch algorithm uses pivot = (min + max) / 2. + // The pivot we use here tries to be smarter and to choose a pivot close to the expectable location of the key. + // This reduces dramatically the number of steps needed to get to the key. + // However, it does not work well with a logaritmic distribution of values, for instance. + // When the key is not found quickly the smart way, we switch to the standard pivot. + if (nPreviousSteps > 2) { + pivot = (min + max) >> 1; + // stop increasing nPreviousSteps from now on + } else { + // NOTE: We cannot do the following operations in int precision, because there might be overflows. + // long operations are slower than float operations with the hardware this was tested on (intel core duo 2, JVM 1.6.0). + // Overall, using float proved to be the safest and fastest approach. + pivot = min + (int)((key - (float)minVal) / (maxVal - (float)minVal) * (max - min)); + nPreviousSteps++; + } + + int pivotVal = array[pivot]; + + // NOTE: do not store key - pivotVal because of overflows + if (key > pivotVal) { + min = pivot + 1; + max--; + } else if (key == pivotVal) { + return pivot; + } else { + min++; + max = pivot - 1; + } + maxVal = array[max]; + minVal = array[min]; + } + } + + static int blockIndex(int n, int[] starts, int numBlocks, int lo) { // find reader for doc n: + int hi = starts.length - 1; // for first element less + while (hi >= lo) { + int mid = (lo + hi) >> 1; + int midValue = starts[mid]; + if (n < midValue) + hi = mid - 1; + else if (n > midValue) + lo = mid + 1; + else { // found a match + while (mid+1 < numBlocks && starts[mid+1] == midValue) { + mid++; // scan to last match + } + return mid; + } + } + return hi; + } +} Index: org/apache/lucene/index/TagTermEnum.java =================================================================== --- org/apache/lucene/index/TagTermEnum.java (revision 0) +++ org/apache/lucene/index/TagTermEnum.java (revision 0) @@ -0,0 +1,6 @@ +package org.apache.lucene.index; + +public abstract class TagTermEnum extends TermEnum { + public abstract int termNum(); + abstract TagTermInfo termInfo(); +} Index: org/apache/lucene/index/TagTermInfo.java =================================================================== --- org/apache/lucene/index/TagTermInfo.java (revision 0) +++ org/apache/lucene/index/TagTermInfo.java (revision 0) @@ -0,0 +1,24 @@ +package org.apache.lucene.index; + +public class TagTermInfo { + int docFreq = 0; + int termNum; + long blockInfosPointer; + long basePointer = 0; + + TagTermInfo() {} + + TagTermInfo(TagTermInfo ti) { + docFreq = ti.docFreq; + termNum = ti.termNum; + blockInfosPointer = ti.blockInfosPointer; + basePointer = ti.basePointer; + } + + final void set(TagTermInfo ti) { + docFreq = ti.docFreq; + termNum = ti.termNum; + blockInfosPointer = ti.blockInfosPointer; + basePointer = ti.basePointer; + } +} Index: org/apache/lucene/index/TagTermInfosReader.java =================================================================== --- org/apache/lucene/index/TagTermInfosReader.java (revision 0) +++ org/apache/lucene/index/TagTermInfosReader.java (revision 0) @@ -0,0 +1,384 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.cache.Cache; +import org.apache.lucene.util.cache.SimpleLRUCache; + +public class TagTermInfosReader { + private Directory directory; + private String segment; + private TagFieldInfos fieldInfos; + + private ThreadLocal threadResources = new ThreadLocal(); + private TagSegmentTermEnum origEnum; + private long size; + + private Term[] indexTerms = null; + private TagTermInfo[] indexInfos; + private long[] indexPointers; + + private TagSegmentTermEnum indexEnum; + + private int indexDivisor = 1; + private int totalIndexInterval; + + private final static int DEFAULT_CACHE_SIZE = 1024; + + /** + * Per-thread resources managed by ThreadLocal + */ + private static final class ThreadResources { + TagSegmentTermEnum termEnum; + + // Used for caching the least recently looked-up Terms + Cache termInfoCache; + } + + TagTermInfosReader(Directory dir, String seg, TagFieldInfos fis) throws CorruptIndexException, IOException { + this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE); + } + + public IndexInput getInput() { + return origEnum.input; + } + + TagTermInfosReader(Directory dir, String seg, TagFieldInfos fis, int readBufferSize) throws CorruptIndexException, IOException { + boolean success = false; + + try { + directory = dir; + segment = seg; + fieldInfos = fis; + + origEnum = new TagSegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, readBufferSize), fieldInfos, + false); + size = origEnum.size; + totalIndexInterval = origEnum.indexInterval; + + indexEnum = new TagSegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, readBufferSize), + fieldInfos, true); + + success = true; + } finally { + // With lock-less commits, it's entirely possible (and + // fine) to hit a FileNotFound exception above. In + // this case, we want to explicitly close any subset + // of things that were opened so that we don't have to + // wait for a GC to do so. + if (!success) { + close(); + } + } + } + + public int getSkipInterval() { + return origEnum.skipInterval; + } + + public int getMaxSkipLevels() { + return origEnum.maxSkipLevels; + } + + /** + *

+ * Sets the indexDivisor, which subsamples the number of indexed terms loaded + * into memory. This has a similar effect as {@link + * IndexWriter#setTermIndexInterval} except that setting must be done at + * indexing time while this setting can be set per reader. When set to N, then + * one in every N*termIndexInterval terms in the index is loaded into memory. + * By setting this to a value > 1 you can reduce memory usage, at the expense + * of higher latency when loading a TermInfo. The default value is 1. + *

+ * + * NOTE: you must call this before the term index is loaded. If the + * index is already loaded, an IllegalStateException is thrown. + + * + * @throws IllegalStateException + * if the term index has already been loaded into memory. + */ + public void setIndexDivisor(int indexDivisor) throws IllegalStateException { + if (indexDivisor < 1) + throw new IllegalArgumentException("indexDivisor must be > 0: got " + indexDivisor); + + if (indexTerms != null) + throw new IllegalStateException("index terms are already loaded"); + + this.indexDivisor = indexDivisor; + totalIndexInterval = origEnum.indexInterval * indexDivisor; + } + + /** + * Returns the indexDivisor. + * + * @see #setIndexDivisor + */ + public int getIndexDivisor() { + return indexDivisor; + } + + final void close() throws IOException { + if (origEnum != null) + origEnum.close(); + if (indexEnum != null) + indexEnum.close(); + threadResources.set(null); + } + + /** Returns the number of term/value pairs in the set. */ + final long size() { + return size; + } + + private ThreadResources getThreadResources() { + ThreadResources resources = (ThreadResources) threadResources.get(); + if (resources == null) { + resources = new ThreadResources(); + resources.termEnum = terms(); + // Cache does not have to be thread-safe, it is only used by one thread at + // the same time + resources.termInfoCache = new SimpleLRUCache(DEFAULT_CACHE_SIZE); + threadResources.set(resources); + } + return resources; + } + + private synchronized void ensureIndexIsRead() throws IOException { + if (indexTerms != null) // index already read + return; // do nothing + try { + int indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor; + + indexTerms = new Term[indexSize]; + indexInfos = new TagTermInfo[indexSize]; + indexPointers = new long[indexSize]; + + for (int i = 0; indexEnum.next(); i++) { + indexTerms[i] = indexEnum.term(); + indexInfos[i] = indexEnum.termInfo(); + indexPointers[i] = indexEnum.indexPointer; + for (int j = 1; j < indexDivisor; j++) + if (!indexEnum.next()) + break; + } + } finally { + indexEnum.close(); + indexEnum = null; + } + } + + /** + * Returns the offset of the greatest index entry which is less than or equal + * to term. + */ + private final int getIndexOffset(int num) { + int lo = 0; // binary search indexTerms[] + int hi = indexTerms.length - 1; + while (hi >= lo) { + int mid = (lo + hi) >> 1; + int indexTermNum = indexInfos[mid].termNum; + int delta = num; // term.compareTo(indexTerms[mid]); + // if (delta < 0) + if (num < indexTermNum) + hi = mid - 1; + else if (num > indexTermNum) + lo = mid + 1; + else + return mid; + } + return hi; + } + + /** + * Returns the offset of the greatest index entry which is less than or equal + * to term. + */ + private final int getIndexOffset(Term term) { + int lo = 0; // binary search indexTerms[] + int hi = indexTerms.length - 1; + + while (hi >= lo) { + int mid = (lo + hi) >> 1; + int delta = term.compareTo(indexTerms[mid]); + if (delta < 0) + hi = mid - 1; + else if (delta > 0) + lo = mid + 1; + else + return mid; + } + return hi; + } + + private final void seekEnum(TagSegmentTermEnum enumerator, int indexOffset) throws IOException { + enumerator.seek(indexPointers[indexOffset], (indexOffset * totalIndexInterval) - 1, indexTerms[indexOffset], indexInfos[indexOffset]); + } + + /** Returns the TermInfo for a Term in the set, or null. */ + TagTermInfo get(Term term) throws IOException { + return get(term, true); + } + + public static class NumResult { + public Term term; + public TagTermInfo tagTermInfo; + } + + NumResult getByNum(int termNum) throws IOException { + if (size == 0) + return null; + ensureIndexIsRead(); + //TagTermInfo ti = null; + NumResult numResult = new NumResult(); + ThreadResources resources = getThreadResources(); + TagSegmentTermEnum enumerator = resources.termEnum; + if (enumerator.term() != null // term is at or past current + && ((enumerator.prev() != null && termNum > enumerator.prevNum()) || termNum >= enumerator.termNum())) { + int enumOffset = (int) (enumerator.position / totalIndexInterval) + 1; + if (indexTerms.length == enumOffset // but before end of block + || termNum < this.indexInfos[enumOffset].termNum) { + // no need to seek + int numScans = enumerator.scanTo(termNum); + if (enumerator.term() != null && termNum == enumerator.termNum()) { + numResult.tagTermInfo = enumerator.termInfo(); + numResult.term = enumerator.term(); + } + } else { + numResult.tagTermInfo = null; + numResult.term = null; + } + return numResult; + } + // random-access: must seek + seekEnum(enumerator, getIndexOffset(termNum)); + enumerator.scanTo(termNum); + if (enumerator.term() != null && termNum == enumerator.termNum()) { + numResult.tagTermInfo = enumerator.termInfo(); + numResult.term = enumerator.term(); + } else { + numResult.tagTermInfo = null; + numResult.term = null; + } + return numResult; + } + + /** Returns the TermInfo for a Term in the set, or null. */ + private TagTermInfo get(Term term, boolean useCache) throws IOException { + if (size == 0) + return null; + + ensureIndexIsRead(); + + TagTermInfo ti; + ThreadResources resources = getThreadResources(); + Cache cache = null; + + if (useCache) { + cache = resources.termInfoCache; + // check the cache first if the term was recently looked up + ti = (TagTermInfo) cache.get(term); + if (ti != null) { + return ti; + } + } + + // optimize sequential access: first try scanning cached enum w/o seeking + TagSegmentTermEnum enumerator = resources.termEnum; + if (enumerator.term() != null // term is at or past current + && ((enumerator.prev() != null && term.compareTo(enumerator.prev()) > 0) || term.compareTo(enumerator.term()) >= 0)) { + int enumOffset = (int) (enumerator.position / totalIndexInterval) + 1; + if (indexTerms.length == enumOffset // but before end of block + || term.compareTo(indexTerms[enumOffset]) < 0) { + // no need to seek + + int numScans = enumerator.scanTo(term); + if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { + ti = enumerator.termInfo(); + if (cache != null && numScans > 1) { + // we only want to put this TermInfo into the cache if + // scanEnum skipped more than one dictionary entry. + // This prevents RangeQueries or WildcardQueries to + // wipe out the cache when they iterate over a large numbers + // of terms in order + cache.put(term, ti); + } + } else { + ti = null; + } + + return ti; + } + } + + // random-access: must seek + seekEnum(enumerator, getIndexOffset(term)); + enumerator.scanTo(term); + if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { + ti = enumerator.termInfo(); + if (cache != null) { + cache.put(term, ti); + } + } else { + ti = null; + } + return ti; + } + + /** Returns the nth term in the set. */ + final Term get(int position) throws IOException { + if (size == 0) + return null; + + TagSegmentTermEnum enumerator = getThreadResources().termEnum; + if (enumerator != null && enumerator.term() != null && position >= enumerator.position + && position < (enumerator.position + totalIndexInterval)) + return scanEnum(enumerator, position); // can avoid seek + + seekEnum(enumerator, position / totalIndexInterval); // must seek + return scanEnum(enumerator, position); + } + + private final Term scanEnum(TagSegmentTermEnum enumerator, int position) throws IOException { + while (enumerator.position < position) + if (!enumerator.next()) + return null; + + return enumerator.term(); + } + + /** Returns the position of a Term in the set or -1. */ + final long getPosition(Term term) throws IOException { + if (size == 0) + return -1; + + ensureIndexIsRead(); + int indexOffset = getIndexOffset(term); + + TagSegmentTermEnum enumerator = getThreadResources().termEnum; + seekEnum(enumerator, indexOffset); + + while (term.compareTo(enumerator.term()) > 0 && enumerator.next()) { + } + + if (term.compareTo(enumerator.term()) == 0) + return enumerator.position; + else + return -1; + } + + /** Returns an enumeration of all the Terms and TermInfos in the set. */ + public TagSegmentTermEnum terms() { + return (TagSegmentTermEnum) origEnum.clone(); + } + + /** Returns an enumeration of terms starting at or after the named term. */ + public TagSegmentTermEnum terms(Term term) throws IOException { + // don't use the cache in this call because we want to reposition the + // enumeration + get(term, false); + return (TagSegmentTermEnum) getThreadResources().termEnum.clone(); + } +} Index: org/apache/lucene/index/TagTermInfosWriter.java =================================================================== --- org/apache/lucene/index/TagTermInfosWriter.java (revision 0) +++ org/apache/lucene/index/TagTermInfosWriter.java (revision 0) @@ -0,0 +1,209 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.UnicodeUtil; + +public class TagTermInfosWriter { + /** The file format version, a negative number. */ + //public static final int FORMAT = 1; + + // Changed strings to true utf8 with length-in-bytes not + // length-in-chars + //public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; + + // NOTE: always change this if you switch to a new format! + public static final int FORMAT_CURRENT = 1; + + private TagFieldInfos fieldInfos; + private IndexOutput output; + //private TagTermInfo lastTi = new TagTermInfo(); + private long size; + + int indexInterval = 128; + int skipInterval = 16; + + /** + * Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + int maxSkipLevels = 10; + + private long lastIndexPointer; + private boolean isIndex; + private byte[] lastTermBytes = new byte[10]; + private int lastTermBytesLength = 0; + private int lastFieldNumber = -1; + private int lastTermNum; + private int lastDocFreq; + private int numBlocks; + private int[] starts; + private int[] docsPerBlock; + + private TagTermInfosWriter other; + private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result(); + + TagTermInfosWriter(Directory directory, String segment, TagFieldInfos fis, int interval) throws IOException { + initialize(directory, segment, fis, interval, false); + other = new TagTermInfosWriter(directory, segment, fis, interval, true); + other.other = this; + } + + private TagTermInfosWriter(Directory directory, String segment, TagFieldInfos fis, int interval, boolean isIndex) throws IOException { + initialize(directory, segment, fis, interval, isIndex); + } + + private void initialize(Directory directory, String segment, TagFieldInfos fis, int interval, boolean isi) throws IOException { + indexInterval = interval; + fieldInfos = fis; + isIndex = isi; + output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis")); + output.writeInt(FORMAT_CURRENT); // write format + output.writeLong(0); // leave space for size + output.writeInt(indexInterval); // write indexInterval + output.writeInt(skipInterval); // write skipInterval + output.writeInt(maxSkipLevels); // write maxSkipLevels + assert initUTF16Results(); + } + + void add(Term term, int termNum, int[] docs) throws IOException { + UnicodeUtil.UTF16toUTF8(term.text, 0, term.text.length(), utf8Result); + add(fieldInfos.fieldNumber(term.field), utf8Result.result, utf8Result.length, termNum, docs.length, docs); + } + + // Currently used only by assert statements + UnicodeUtil.UTF16Result utf16Result1; + UnicodeUtil.UTF16Result utf16Result2; + + // Currently used only by assert statements + private boolean initUTF16Results() { + utf16Result1 = new UnicodeUtil.UTF16Result(); + utf16Result2 = new UnicodeUtil.UTF16Result(); + return true; + } + + // Currently used only by assert statement + private int compareToLastTerm(int fieldNumber, byte[] termBytes, int termBytesLength) { + if (lastFieldNumber != fieldNumber) { + final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); + if (cmp != 0 || lastFieldNumber != -1) + return cmp; + } + UnicodeUtil.UTF8toUTF16(lastTermBytes, 0, lastTermBytesLength, utf16Result1); + UnicodeUtil.UTF8toUTF16(termBytes, 0, termBytesLength, utf16Result2); + final int len; + if (utf16Result1.length < utf16Result2.length) + len = utf16Result1.length; + else + len = utf16Result2.length; + + for (int i = 0; i < len; i++) { + final char ch1 = utf16Result1.result[i]; + final char ch2 = utf16Result2.result[i]; + if (ch1 != ch2) + return ch1 - ch2; + } + return utf16Result1.length - utf16Result2.length; + } + + /** + * Adds a new <, TermInfo> pair to the set. Term must + * be lexicographically greater than all previous Terms added. TermInfo + * pointers must be positive and greater than all previous. + */ + void add(int fieldNumber, byte[] termBytes, int termBytesLength, int termNum, int docFreq, int[] docs) throws IOException { + assert compareToLastTerm(fieldNumber, termBytes, termBytesLength) < 0 || (isIndex && termBytesLength == 0 && lastTermBytesLength == 0) : "Terms are out of order: field=" + + fieldInfos.fieldName(fieldNumber) + + " (number " + + fieldNumber + + ")" + + " lastField=" + + fieldInfos.fieldName(lastFieldNumber) + + " (number " + + lastFieldNumber + + ")" + + " text=" + + new String(termBytes, 0, termBytesLength, "UTF-8") + + " lastText=" + + new String(lastTermBytes, 0, lastTermBytesLength, "UTF-8"); + + //assert ti.freqPointer >= lastTi.freqPointer : "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; + //assert ti.proxPointer >= lastTi.proxPointer : "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; + + if (!isIndex && size % indexInterval == 0) + other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTermNum, lastDocFreq, null); + + writeTerm(fieldNumber, termBytes, termBytesLength); // write term + + output.writeVInt(docFreq); // write doc freq + output.writeVInt(termNum); + if (!isIndex) { + TagBlockInfos tagBlockInfos = new TagBlockInfos(); + tagBlockInfos.termNum = termNum; + + TagBlocksData tagBlocksData = new TagBlocksData(termNum, docs, skipInterval, maxSkipLevels, starts, docsPerBlock); + tagBlockInfos.docFreqs = tagBlocksData.getDocFreqs(); + tagBlockInfos.positions = tagBlocksData.getPositions(); + RAMOutputStream blockInfosOutput = new RAMOutputStream(); + tagBlockInfos.write(blockInfosOutput); + output.writeVInt(tagBlocksData.length()); + output.writeVInt((int)blockInfosOutput.length()); + tagBlocksData.write(output); + blockInfosOutput.writeTo(output); + } + + //output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers + //output.writeVLong(ti.proxPointer - lastTi.proxPointer); + + //if (ti.docFreq >= skipInterval) { + // output.writeVInt(ti.skipOffset); + //} + + if (isIndex) { + output.writeVLong(other.output.getFilePointer() - lastIndexPointer); + lastIndexPointer = other.output.getFilePointer(); // write pointer + } + lastFieldNumber = fieldNumber; + //lastTi.set(ti); + lastTermNum = termNum; + lastDocFreq = docFreq; + size++; + } + + private void writeTerm(int fieldNumber, byte[] termBytes, int termBytesLength) throws IOException { + // TODO: UTF16toUTF8 could tell us this prefix + // Compute prefix in common with last term: + int start = 0; + final int limit = termBytesLength < lastTermBytesLength ? termBytesLength : lastTermBytesLength; + while (start < limit) { + if (termBytes[start] != lastTermBytes[start]) + break; + start++; + } + final int length = termBytesLength - start; + output.writeVInt(start); // write shared prefix length + output.writeVInt(length); // write delta length + output.writeBytes(termBytes, start, length); // write delta bytes + output.writeVInt(fieldNumber); // write field num + if (lastTermBytes.length < termBytesLength) { + byte[] newArray = new byte[(int) (termBytesLength * 1.5)]; + System.arraycopy(lastTermBytes, 0, newArray, 0, start); + lastTermBytes = newArray; + } + System.arraycopy(termBytes, start, lastTermBytes, start, length); + lastTermBytesLength = termBytesLength; + } + + /** Called to complete TermInfos creation. */ + void close() throws IOException { + output.seek(4); // write size after format + output.writeLong(size); + output.close(); + + if (!isIndex) + other.close(); + } +} Index: org/apache/lucene/index/TagUtil.java =================================================================== --- org/apache/lucene/index/TagUtil.java (revision 0) +++ org/apache/lucene/index/TagUtil.java (revision 0) @@ -0,0 +1,15 @@ +package org.apache.lucene.index; + +public class TagUtil { + + public static boolean toBoolean(byte b) { + if (b == 0) return false; + else return true; + } + + public static byte toByte(boolean value) { + byte b = 0; + if (value) b = 1; + return b; + } +} Index: org/apache/lucene/index/TagWriteable.java =================================================================== --- org/apache/lucene/index/TagWriteable.java (revision 0) +++ org/apache/lucene/index/TagWriteable.java (revision 0) @@ -0,0 +1,9 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.store.IndexOutput; + +public interface TagWriteable { + public void write(IndexOutput output) throws IOException; +} Index: org/apache/lucene/store/ByteArrayIndexInput.java =================================================================== --- org/apache/lucene/store/ByteArrayIndexInput.java (revision 0) +++ org/apache/lucene/store/ByteArrayIndexInput.java (revision 0) @@ -0,0 +1,51 @@ +package org.apache.lucene.store; + +public class ByteArrayIndexInput extends IndexInput { + private int position = 0; + private byte[] bytes; + + public ByteArrayIndexInput(byte[] bytes) { + this.bytes = bytes; + } + + public long length() { + return bytes.length; + } + + public void readBytes(byte[] b, int offset, int len) { + if (bytes == null) { + throw new NullPointerException(); + } else if (offset < 0 || len < 0 || len > b.length - offset) { + throw new IndexOutOfBoundsException(); + } + //if (position >= bytes.length) { + // return -1; + //} + if (position + len > b.length) { + len = bytes.length - position; + } + //if (len <= 0) { + // return 0; + //} + System.arraycopy(bytes, position, b, offset, len); + //position += len; + //return len; + } + + public byte readByte() { + byte b = bytes[position]; + position++; + return b; + } + + public long getFilePointer() { + return position; + } + + public void seek(long position) { + this.position = (int) position; + } + + public void close() { + } +} Index: org/apache/lucene/store/ByteArrayIndexOutput.java =================================================================== --- org/apache/lucene/store/ByteArrayIndexOutput.java (revision 0) +++ org/apache/lucene/store/ByteArrayIndexOutput.java (revision 0) @@ -0,0 +1,52 @@ +package org.apache.lucene.store; + +/** + * + * + */ +// TODO: make resizable +public class ByteArrayIndexOutput extends IndexOutput { + private byte[] bytes; + private int position = 0; + + public ByteArrayIndexOutput() { + + } + + public ByteArrayIndexOutput(int length) { + bytes = new byte[length]; + } + + public byte[] toByteArray() { + return bytes; + } + + public void flush() {} + + public long getFilePointer() { + return position; + } + + public void writeBytes(byte[] b, int offset, int length) { + System.arraycopy(b, offset, bytes, position, length); + } + + public void writeBytes(byte[] b, int length) { + System.arraycopy(b, 0, bytes, position, length); + } + + public void writeByte(byte b) { + bytes[position] = b; + position++; + } + + public void seek(long pointer) { + throw new UnsupportedOperationException(""); + } + + public void close() {} + + public long length() { + return bytes.length; + } +} Index: org/apache/lucene/store/LimitedIndexInput.java =================================================================== --- org/apache/lucene/store/LimitedIndexInput.java (revision 0) +++ org/apache/lucene/store/LimitedIndexInput.java (revision 0) @@ -0,0 +1,49 @@ +package org.apache.lucene.store; + +import java.io.IOException; + +public class LimitedIndexInput extends IndexInput { + private long contentLength; + private long pos = 0; + private IndexInput input; + private long startPosition; + + public LimitedIndexInput(int contentLength, IndexInput input) { + startPosition = input.getFilePointer(); + this.contentLength = contentLength; + this.input = input; + } + + public void close() {} + + public void seek(long value) throws IOException { + input.seek(startPosition + value); + } + + public void readBytes(byte[] b, int off, int len) throws IOException { + if (pos >= contentLength) { + return; + } + if (pos + len > contentLength) { + len = (int) (contentLength - pos); + } + input.readBytes(b, off, len); + pos += len; + } + + public long length() { + return contentLength; + } + + public byte readByte() throws IOException { + if (pos >= contentLength) { + return -1; + } + pos++; + return input.readByte(); + } + + public long getFilePointer() { + return pos; + } +} Index: org/apache/lucene/store/RAMOutputStream.java =================================================================== --- org/apache/lucene/store/RAMOutputStream.java (revision 664366) +++ org/apache/lucene/store/RAMOutputStream.java (working copy) @@ -28,7 +28,7 @@ public class RAMOutputStream extends IndexOutput { static final int BUFFER_SIZE = 1024; - private RAMFile file; + protected RAMFile file; private byte[] currentBuffer; private int currentBufferIndex; Index: org/apache/lucene/store/TagRAMOutputStream.java =================================================================== --- org/apache/lucene/store/TagRAMOutputStream.java (revision 0) +++ org/apache/lucene/store/TagRAMOutputStream.java (revision 0) @@ -0,0 +1,12 @@ +package org.apache.lucene.store; + +import java.io.IOException; + +/** + * + */ +public class TagRAMOutputStream extends RAMOutputStream { + public IndexInput getInput() throws IOException { + return new RAMInputStream(file); + } +}