diff --git a/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java b/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java index a696505..e45248c 100644 --- a/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java +++ b/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java @@ -317,10 +317,8 @@ public class IndexDocValuesField extends Field implements PerDocFieldValues { final String value; switch (type) { case BYTES_FIXED_DEREF: - case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: value = "bytes:bytes.utf8ToString();"; break; @@ -353,10 +351,8 @@ public class IndexDocValuesField extends Field implements PerDocFieldValues { final IndexDocValuesField valField = new IndexDocValuesField(field.name(), field.fieldType(), field.stringValue()); switch (type) { case BYTES_FIXED_DEREF: - case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: BytesRef ref = field.isBinary() ? field.binaryValue() : new BytesRef(field.stringValue()); valField.setBytes(ref, type); diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 2982203..2ad0c24 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -41,7 +41,7 @@ import java.util.Map; import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; -import org.apache.lucene.index.values.ValuesEnum; +import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -1070,27 +1070,26 @@ public class CheckIndex { if (docValues == null) { continue; } - final ValuesEnum values = docValues.getEnum(); - while (values.nextDoc() != ValuesEnum.NO_MORE_DOCS) { + final Source values = docValues.getDirectSource(); + final int maxDoc = reader.maxDoc(); + for (int i = 0; i < maxDoc; i++) { switch (fieldInfo.docValues) { case BYTES_FIXED_DEREF: - case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: - values.bytes(); + values.getBytes(i, new BytesRef()); break; case FLOAT_32: case FLOAT_64: - values.getFloat(); + values.getFloat(i); break; case VAR_INTS: case FIXED_INTS_16: case FIXED_INTS_32: case FIXED_INTS_64: case FIXED_INTS_8: - values.getInt(); + values.getInt(i); break; default: throw new IllegalArgumentException("Field: " + fieldInfo.name diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index 6108aeb..c8a0646 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -652,29 +652,23 @@ public final class FieldInfos implements Iterable { case BYTES_FIXED_DEREF: b = 5; break; - case BYTES_FIXED_SORTED: - b = 6; - break; case BYTES_VAR_STRAIGHT: - b = 7; + b = 6; break; case BYTES_VAR_DEREF: - b = 8; - break; - case BYTES_VAR_SORTED: - b = 9; + b = 7; break; case FIXED_INTS_16: - b = 10; + b = 8; break; case FIXED_INTS_32: - b = 11; + b = 9; break; case FIXED_INTS_64: - b = 12; + b = 10; break; case FIXED_INTS_8: - b = 13; + b = 11; break; default: @@ -754,27 +748,21 @@ public final class FieldInfos implements Iterable { docValuesType = ValueType.BYTES_FIXED_DEREF; break; case 6: - docValuesType = ValueType.BYTES_FIXED_SORTED; - break; - case 7: docValuesType = ValueType.BYTES_VAR_STRAIGHT; break; - case 8: + case 7: docValuesType = ValueType.BYTES_VAR_DEREF; break; - case 9: - docValuesType = ValueType.BYTES_VAR_SORTED; - break; - case 10: + case 8: docValuesType = ValueType.FIXED_INTS_16; break; - case 11: + case 9: docValuesType = ValueType.FIXED_INTS_32; break; - case 12: + case 10: docValuesType = ValueType.FIXED_INTS_64; break; - case 13: + case 11: docValuesType = ValueType.FIXED_INTS_8; break; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java index 0e1e21d..dbf6ec2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java @@ -20,7 +20,6 @@ package org.apache.lucene.index.codecs; import java.io.Closeable; import java.io.IOException; import java.util.Collection; -import java.util.Comparator; import java.util.Map; import java.util.TreeMap; @@ -33,7 +32,6 @@ import org.apache.lucene.index.values.Ints; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; /** * Abstract base class for PerDocValues implementations @@ -59,10 +57,6 @@ public abstract class DocValuesReaderBase extends PerDocValues { return docValues().keySet(); } - public Comparator getComparator() throws IOException { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, String segment, int docCount, Directory dir, int codecId, IOContext context) @@ -125,17 +119,13 @@ public abstract class DocValuesReaderBase extends PerDocValues { case FLOAT_64: return Floats.getValues(dir, id, docCount, context); case BYTES_FIXED_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context); + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, context); case BYTES_FIXED_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context); - case BYTES_FIXED_SORTED: - return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context); + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, context); case BYTES_VAR_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context); + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, context); case BYTES_VAR_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context); - case BYTES_VAR_SORTED: - return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context); + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, context); default: throw new IllegalStateException("unrecognized index values mode " + type); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java index 21cf563..bc37527 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java @@ -18,14 +18,12 @@ package org.apache.lucene.index.codecs; */ import java.io.IOException; -import java.util.Comparator; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.values.Writer; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; /** @@ -55,14 +53,10 @@ public abstract class DocValuesWriterBase extends PerDocConsumer { public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), docValuesId(segmentName, codecId, field.number), - getDirectory(), getComparator(), bytesUsed, context); + getDirectory(), bytesUsed, context); } public static String docValuesId(String segmentsName, int codecID, int fieldId) { return segmentsName + "_" + codecID + "-" + fieldId; } - - public Comparator getComparator() throws IOException { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java index 1419950..2bdae4b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java @@ -56,8 +56,6 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: - case BYTES_FIXED_SORTED: case BYTES_VAR_STRAIGHT: files.add(IndexFileNames.segmentFileName(filename, "", Writer.INDEX_EXTENSION)); diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index 70f32fa..a054767 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -20,20 +20,16 @@ package org.apache.lucene.index.values; /** Base class for specific Bytes Reader/Writer implementations */ import java.io.IOException; import java.util.Collection; -import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; -import org.apache.lucene.index.values.IndexDocValues.SourceEnum; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefHash; @@ -84,10 +80,6 @@ public final class Bytes { * Mode for dereferenced stored bytes */ DEREF, - /** - * Mode for sorted stored bytes - */ - SORTED }; /** @@ -116,30 +108,22 @@ public final class Bytes { * @throws IOException * if the files for the writer can not be created. */ - public static Writer getWriter(Directory dir, String id, Mode mode, - Comparator comp, boolean fixedSize, Counter bytesUsed, IOContext context) + public static Writer getWriter(Directory dir, String id, Mode mode, boolean fixedSize, Counter bytesUsed, IOContext context) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? - if (comp == null) { - comp = BytesRef.getUTF8SortedAsUnicodeComparator(); - } if (fixedSize) { if (mode == Mode.STRAIGHT) { return new FixedStraightBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); - } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context); } } else { if (mode == Mode.STRAIGHT) { return new VarStraightBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); - } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context); } } @@ -169,7 +153,7 @@ public final class Bytes { * if an {@link IOException} occurs */ public static IndexDocValues getValues(Directory dir, String id, Mode mode, - boolean fixedSize, int maxDoc, Comparator sortComparator, IOContext context) throws IOException { + boolean fixedSize, int maxDoc, IOContext context) throws IOException { // TODO -- I can peek @ header to determing fixed/mode? if (fixedSize) { @@ -177,16 +161,12 @@ public final class Bytes { return new FixedStraightBytesImpl.Reader(dir, id, maxDoc, context); } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Reader(dir, id, maxDoc, context); - } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Reader(dir, id, maxDoc, context); } } else { if (mode == Mode.STRAIGHT) { return new VarStraightBytesImpl.Reader(dir, id, maxDoc, context); } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Reader(dir, id, maxDoc, context); - } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator, context); } } @@ -249,24 +229,6 @@ public final class Bytes { */ protected abstract int maxDoc(); - @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return new SourceEnum(attrSource, type(), this, maxDoc()) { - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - while (source.getBytes(target, bytesRef).length == 0) { - if (++target >= numDocs) { - return pos = NO_MORE_DOCS; - } - } - return pos = target; - } - }; - } - } static abstract class DerefBytesSourceBase extends BytesSourceBase { @@ -288,118 +250,13 @@ public final class Bytes { } - static abstract class BytesSortedSourceBase extends SortedSource { - private final PagedBytes pagedBytes; - private final Comparator comp; - protected final PackedInts.Reader docToOrdIndex; - private final ValueType type; - - protected final IndexInput datIn; - protected final IndexInput idxIn; - protected final BytesRef defaultValue = new BytesRef(); - protected final static int PAGED_BYTES_BITS = 15; - protected final PagedBytes.Reader data; - - - protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn, - Comparator comp, long bytesToRead, ValueType type) throws IOException { - this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type); - } - - protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn, - Comparator comp, PagedBytes pagedBytes, long bytesToRead,ValueType type) - throws IOException { - assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: " - + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer(); - this.datIn = datIn; - this.pagedBytes = pagedBytes; - this.pagedBytes.copy(datIn, bytesToRead); - data = pagedBytes.freeze(true); - this.idxIn = idxIn; - this.comp = comp == null ? BytesRef.getUTF8SortedAsUnicodeComparator() - : comp; - docToOrdIndex = PackedInts.getReader(idxIn); - this.type = type; - - } - - @Override - public int ord(int docID) { - return (int) docToOrdIndex.get(docID) -1; - } - - @Override - public BytesRef getByOrd(int ord, BytesRef bytesRef) { - assert ord >= 0; - return deref(ord, bytesRef); - } - - protected void closeIndexInput() throws IOException { - IOUtils.close(datIn, idxIn); - } - - /** - * Returns the largest doc id + 1 in this doc values source - */ - public int maxDoc() { - return docToOrdIndex.size(); - } - /** - * Copies the value for the given ord to the given {@link BytesRef} and - * returns it. - */ - protected abstract BytesRef deref(int ord, BytesRef bytesRef); - - protected int binarySearch(BytesRef b, BytesRef bytesRef, int low, - int high) { - int mid = 0; - while (low <= high) { - mid = (low + high) >>> 1; - deref(mid, bytesRef); - final int cmp = comp.compare(bytesRef, b); - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return mid; - } - } - assert comp.compare(bytesRef, b) != 0; - return -(low + 1); - } - - @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return new SourceEnum(attrSource, type(), this, maxDoc()) { - - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - while (source.getBytes(target, bytesRef).length == 0) { - if (++target >= numDocs) { - return pos = NO_MORE_DOCS; - } - } - return pos = target; - } - }; - } - - @Override - public ValueType type() { - return type; - } - } // TODO: open up this API?! static abstract class BytesWriterBase extends Writer { private final String id; private IndexOutput idxOut; private IndexOutput datOut; - protected BytesRef bytesRef; + protected BytesRef bytesRef = new BytesRef(); private final Directory dir; private final String codecName; private final int version; @@ -467,8 +324,8 @@ public final class Bytes { public abstract void finish(int docCount) throws IOException; @Override - protected void mergeDoc(int docID) throws IOException { - add(docID, bytesRef); + protected void mergeDoc(int docID, int sourceDoc) throws IOException { + add(docID, currentMergeSource.getBytes(sourceDoc, bytesRef)); } @Override @@ -480,11 +337,6 @@ public final class Bytes { } @Override - protected void setNextEnum(ValuesEnum valuesEnum) { - bytesRef = valuesEnum.bytes(); - } - - @Override public void files(Collection files) throws IOException { assert datOut != null; files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION)); @@ -570,6 +422,7 @@ public final class Bytes { static abstract class DerefBytesWriterBase extends BytesWriterBase { protected int size = -1; + protected int lastDocId = -1; protected int[] docToEntry; protected final BytesRefHash hash; @@ -608,17 +461,33 @@ public final class Bytes { return; } checkSize(bytes); + fillDefault(docID); int ord = hash.add(bytes); if (ord < 0) { ord = (-ord) - 1; } + + docToEntry[docID] = ord; + lastDocId = docID; + } + + protected void fillDefault(int docID) { if (docID >= docToEntry.length) { final int size = docToEntry.length; docToEntry = ArrayUtil.grow(docToEntry, 1 + docID); bytesUsed.addAndGet((docToEntry.length - size) * RamUsageEstimator.NUM_BYTES_INT); } - docToEntry[docID] = 1 + ord; + assert size >= 0; + BytesRef ref = new BytesRef(size); + ref.length = size; + int ord = hash.add(ref); + if (ord < 0) { + ord = (-ord) - 1; + } + for (int i = lastDocId+1; i < docID; i++) { + docToEntry[i] = ord; + } } protected void checkSize(BytesRef bytes) { @@ -712,78 +581,4 @@ public final class Bytes { } } - - abstract static class DerefBytesEnumBase extends ValuesEnum { - private final PackedInts.ReaderIterator idx; - private final int valueCount; - private int pos = -1; - protected final IndexInput datIn; - protected final long fp; - protected final int size; - - protected DerefBytesEnumBase(AttributeSource source, IndexInput datIn, - IndexInput idxIn, int size, ValueType enumType) throws IOException { - super(source, enumType); - this.datIn = datIn; - this.size = size; - idx = PackedInts.getReaderIterator(idxIn); - fp = datIn.getFilePointer(); - if (size > 0) { - bytesRef.grow(this.size); - bytesRef.length = this.size; - } - bytesRef.offset = 0; - valueCount = idx.size(); - } - - protected void copyFrom(ValuesEnum valuesEnum) { - bytesRef = valuesEnum.bytesRef; - if (bytesRef.bytes.length < size) { - bytesRef.grow(size); - } - bytesRef.length = size; - bytesRef.offset = 0; - } - - @Override - public int advance(int target) throws IOException { - if (target < valueCount) { - long address; - while ((address = idx.advance(target)) == 0) { - if (++target >= valueCount) { - return pos = NO_MORE_DOCS; - } - } - pos = idx.ord(); - fill(address, bytesRef); - return pos; - } - return pos = NO_MORE_DOCS; - } - - @Override - public int nextDoc() throws IOException { - if (pos >= valueCount) { - return pos = NO_MORE_DOCS; - } - return advance(pos + 1); - } - - public void close() throws IOException { - try { - datIn.close(); - } finally { - idx.close(); - } - } - - protected abstract void fill(long address, BytesRef ref) throws IOException; - - @Override - public int docID() { - return pos; - } - - } - -} \ No newline at end of file +} diff --git a/lucene/src/java/org/apache/lucene/index/values/DirectSource.java b/lucene/src/java/org/apache/lucene/index/values/DirectSource.java new file mode 100644 index 0000000..021f449 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/values/DirectSource.java @@ -0,0 +1,153 @@ +package org.apache.lucene.index.values; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.values.IndexDocValues.Source; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; + +/** + * @lucene.internal + */ +public abstract class DirectSource extends Source{ + + protected final IndexInput data; + private final ValueType type; + private final OffsetAndSize offsetAndSize = new OffsetAndSize(); + private final ToNumeric toNumeric; + protected final long baseOffset; + + DirectSource(IndexInput input, ValueType type) { + this.data = input; + this.type = type; + baseOffset = input.getFilePointer(); + switch (type) { + case FIXED_INTS_16: + toNumeric = new ShortToLong(); + break; + case FLOAT_32: + case FIXED_INTS_32: + toNumeric = new IntToLong(); + break; + case FIXED_INTS_8: + toNumeric = new ByteToLong(); + break; + default: + toNumeric = new LongToLong(); + } + } + + @Override + public BytesRef getBytes(int docID, BytesRef ref) { + + try { + offsetAndSize(docID, offsetAndSize); + data.seek(offsetAndSize.offset + baseOffset); + ref.grow(offsetAndSize.size); + data.readBytes(ref.bytes, 0, offsetAndSize.size); + ref.length = offsetAndSize.size; + ref.offset = 0; + return ref; + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public long getInt(int docID) { + try { + offsetAndSize(docID, offsetAndSize); + data.seek(offsetAndSize.offset + baseOffset); + return toNumeric.toLong(data); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public double getFloat(int docID) { + try { + offsetAndSize(docID, offsetAndSize); + data.seek(offsetAndSize.offset + baseOffset); + return toNumeric.toDouble(data); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + protected abstract void offsetAndSize(int docID, OffsetAndSize offsetAndSize) + throws IOException; + + @Override + public ValueType type() { + return type; + } + + + public static class OffsetAndSize { + long offset; + int size; + } + + private abstract static class ToNumeric { + abstract long toLong(IndexInput input) throws IOException; + + double toDouble(IndexInput input) throws IOException { + return toLong(input); + } + } + + private static final class ByteToLong extends ToNumeric { + @Override + long toLong(IndexInput input) throws IOException { + return input.readByte(); + } + + } + + private static final class ShortToLong extends ToNumeric { + @Override + long toLong(IndexInput input) throws IOException { + return input.readShort(); + } + } + + private static final class IntToLong extends ToNumeric { + @Override + long toLong(IndexInput input) throws IOException { + return input.readInt(); + } + + double toDouble(IndexInput input) throws IOException { + return Float.intBitsToFloat(input.readInt()); + } + } + + private static final class LongToLong extends ToNumeric { + @Override + long toLong(IndexInput input) throws IOException { + return input.readLong(); + } + + double toDouble(IndexInput input) throws IOException { + return Double.longBitsToDouble(input.readLong()); + } + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java index f6ac7ab..0329e0a 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java @@ -21,15 +21,15 @@ import java.io.IOException; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase; -import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase; import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; +import org.apache.lucene.index.values.DirectSource; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; // Stores fixed-length byte[] by deref, ie when two docs // have the same value, they store only 1 byte[] @@ -91,38 +91,39 @@ class FixedDerefBytesImpl { @Override public BytesRef getBytes(int docID, BytesRef bytesRef) { final int id = (int) addresses.get(docID); - if (id == 0) { - bytesRef.length = 0; - return bytesRef; - } - return data.fillSlice(bytesRef, ((id - 1) * size), size); + return data.fillSlice(bytesRef, (id * size), size); } } @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new DerefBytesEnum(source, cloneData(), cloneIndex(), size); + public ValueType type() { + return ValueType.BYTES_FIXED_DEREF; } - final static class DerefBytesEnum extends DerefBytesEnumBase { - - public DerefBytesEnum(AttributeSource source, IndexInput datIn, - IndexInput idxIn, int size) throws IOException { - super(source, datIn, idxIn, size, ValueType.BYTES_FIXED_DEREF); - } + @Override + public org.apache.lucene.index.values.IndexDocValues.Source getDirectSource() + throws IOException { + return new DirectFixedDerefSource(cloneData(), cloneIndex(), size, type()); + } + } + + public final static class DirectFixedDerefSource extends DirectSource { + private PackedInts.SeekableReaderIterator index; + private final int size; - protected void fill(long address, BytesRef ref) throws IOException { - datIn.seek(fp + ((address - 1) * size)); - datIn.readBytes(ref.bytes, 0, size); - ref.length = size; - ref.offset = 0; - } + DirectFixedDerefSource(IndexInput data, IndexInput index, int size, ValueType type) + throws IOException { + super(data, type); + this.size = size; + this.index = PackedInts.getSeekableReaderIterator(index); } @Override - public ValueType type() { - return ValueType.BYTES_FIXED_DEREF; + protected void offsetAndSize(int docID, OffsetAndSize offsetAndSize) + throws IOException { + offsetAndSize.offset = index.seek(docID) * size; + offsetAndSize.size = size; } } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java deleted file mode 100644 index b5969ae..0000000 --- a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java +++ /dev/null @@ -1,143 +0,0 @@ -package org.apache.lucene.index.values; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase; -import org.apache.lucene.index.values.Bytes.BytesReaderBase; -import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; -import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Counter; - -// Stores fixed-length byte[] by deref, ie when two docs -// have the same value, they store only 1 byte[] - -/** - * @lucene.experimental - */ -class FixedSortedBytesImpl { - - static final String CODEC_NAME = "FixedSortedBytes"; - static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; - - static class Writer extends DerefBytesWriterBase { - private final Comparator comp; - - public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); - this.comp = comp; - } - - // Important that we get docCount, in case there were - // some last docs that we didn't see - @Override - public void finishInternal(int docCount) throws IOException { - final IndexOutput datOut = getOrCreateDataOut(); - final int count = hash.size(); - final int[] address = new int[count+1]; // addr 0 is default values - datOut.writeInt(size); - if (size != -1) { - final int[] sortedEntries = hash.sort(comp); - // first dump bytes data, recording address as we go - final BytesRef bytesRef = new BytesRef(size); - for (int i = 0; i < count; i++) { - final int e = sortedEntries[i]; - final BytesRef bytes = hash.get(e, bytesRef); - assert bytes.length == size; - datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length); - address[e + 1] = 1 + i; - } - } - final IndexOutput idxOut = getOrCreateIndexOut(); - idxOut.writeInt(count); - writeIndex(idxOut, docCount, count, address, docToEntry); - } - } - - public static class Reader extends BytesReaderBase { - private final int size; - private final int numValuesStored; - - public Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true, context); - size = datIn.readInt(); - numValuesStored = idxIn.readInt(); - } - - @Override - public org.apache.lucene.index.values.IndexDocValues.Source load() - throws IOException { - return loadSorted(null); - } - - @Override - public SortedSource loadSorted(Comparator comp) - throws IOException { - return new Source(cloneData(), cloneIndex(), size, numValuesStored, comp); - } - - private static class Source extends BytesSortedSourceBase { - private final int valueCount; - private final int size; - - public Source(IndexInput datIn, IndexInput idxIn, int size, - int numValues, Comparator comp) throws IOException { - super(datIn, idxIn, comp, size * numValues, ValueType.BYTES_FIXED_SORTED); - this.size = size; - this.valueCount = numValues; - closeIndexInput(); - } - - @Override - public int getByValue(BytesRef bytes, BytesRef tmpRef) { - return binarySearch(bytes, tmpRef, 0, valueCount - 1); - } - - @Override - public int getValueCount() { - return valueCount; - } - - @Override - protected BytesRef deref(int ord, BytesRef bytesRef) { - return data.fillSlice(bytesRef, (ord * size), size); - } - } - - @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - // do unsorted - return new DerefBytesEnum(source, cloneData(), cloneIndex(), size); - } - - @Override - public ValueType type() { - return ValueType.BYTES_FIXED_SORTED; - } - } -} diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java index 64357b0..bd4dfd9 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java @@ -24,11 +24,11 @@ import java.io.IOException; import org.apache.lucene.index.values.Bytes.BytesSourceBase; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; +import org.apache.lucene.index.values.DirectSource; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator; import org.apache.lucene.util.BytesRef; @@ -175,8 +175,9 @@ class FixedStraightBytesImpl { } @Override - protected void mergeDoc(int docID) throws IOException { + protected void mergeDoc(int docID, int sourceDoc) throws IOException { assert lastDocID < docID; + currentMergeSource.getBytes(sourceDoc, bytesRef); if (size == -1) { size = bytesRef.length; datOut.writeInt(size); @@ -263,11 +264,9 @@ class FixedStraightBytesImpl { // specialized version for single bytes private static class SingleByteSource extends Source { - private final int maxDoc; private final byte[] data; public SingleByteSource(IndexInput datIn, int maxDoc) throws IOException { - this.maxDoc = maxDoc; try { data = new byte[maxDoc]; datIn.readBytes(data, 0, data.length, false); @@ -290,22 +289,6 @@ class FixedStraightBytesImpl { return ValueType.BYTES_FIXED_STRAIGHT; } - @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return new SourceEnum(attrSource, type(), this, maxDoc) { - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - bytesRef.length = 1; - bytesRef.bytes = data; - bytesRef.offset = target; - return pos = target; - } - }; - } - } private final static class StraightBytesSource extends BytesSourceBase { @@ -335,73 +318,32 @@ class FixedStraightBytesImpl { } } - @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new FixedStraightBytesEnum(source, cloneData(), size, maxDoc); - } - @Override public ValueType type() { return ValueType.BYTES_FIXED_STRAIGHT; } + + @Override + public Source getDirectSource() throws IOException { + return new DirectFixedSource(cloneData(), size, type()); + } } - static class FixedStraightBytesEnum extends ValuesEnum { - private final IndexInput datIn; + public final static class DirectFixedSource extends DirectSource { private final int size; - private final int maxDoc; - private int pos = -1; - private final long fp; - - public FixedStraightBytesEnum(AttributeSource source, IndexInput datIn, - int size, int maxDoc) throws IOException { - super(source, ValueType.BYTES_FIXED_STRAIGHT); - this.datIn = datIn; - this.size = size; - this.maxDoc = maxDoc; - bytesRef.grow(size); - bytesRef.length = size; - bytesRef.offset = 0; - fp = datIn.getFilePointer(); - } - protected void copyFrom(ValuesEnum valuesEnum) { - super.copyFrom(valuesEnum); - if (bytesRef.bytes.length < size) { - bytesRef.grow(size); - } - bytesRef.length = size; - bytesRef.offset = 0; - } - - public void close() throws IOException { - datIn.close(); - } - - @Override - public int advance(int target) throws IOException { - if (target >= maxDoc || size == 0) { - return pos = NO_MORE_DOCS; - } - if ((target - 1) != pos) // pos inc == 1 - datIn.seek(fp + target * size); - datIn.readBytes(bytesRef.bytes, 0, size); - return pos = target; + DirectFixedSource(IndexInput input, int size, ValueType type) { + super(input, type); + this.size = size; } - @Override - public int docID() { - return pos; + protected void offsetAndSize(int docID, OffsetAndSize offsetAndSize) + throws IOException { + offsetAndSize.offset = size * docID; + offsetAndSize.size = size; } - @Override - public int nextDoc() throws IOException { - if (pos >= maxDoc) { - return pos = NO_MORE_DOCS; - } - return advance(pos + 1); - } } } diff --git a/lucene/src/java/org/apache/lucene/index/values/Floats.java b/lucene/src/java/org/apache/lucene/index/values/Floats.java index a3c7d57..5ad0a0e 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Floats.java +++ b/lucene/src/java/org/apache/lucene/index/values/Floats.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -101,11 +100,6 @@ public class Floats { } } - public ValuesEnum getEnum(AttributeSource source) throws IOException { - IndexInput indexInput = (IndexInput) datIn.clone(); - return arrayTemplate.getDirectEnum(source, indexInput, maxDoc); - } - @Override public ValueType type() { return arrayTemplate.type(); diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java index 305a076..0ccb553 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java @@ -64,27 +64,6 @@ public abstract class IndexDocValues implements Closeable { private SourceCache cache = new SourceCache.DirectSourceCache(); /** - * Returns an iterator that steps through all documents values for this - * {@link IndexDocValues} field instance. {@link ValuesEnum} will skip document - * without a value if applicable. - */ - public ValuesEnum getEnum() throws IOException { - return getEnum(null); - } - - /** - * Returns an iterator that steps through all documents values for this - * {@link IndexDocValues} field instance. {@link ValuesEnum} will skip document - * without a value if applicable. - *

- * If an {@link AttributeSource} is supplied to this method the - * {@link ValuesEnum} will use the given source to access implementation - * related attributes. - */ - public abstract ValuesEnum getEnum(AttributeSource attrSource) - throws IOException; - - /** * Loads a new {@link Source} instance for this {@link IndexDocValues} field * instance. Source instances returned from this method are not cached. It is * the callers responsibility to maintain the instance and release its @@ -118,56 +97,10 @@ public abstract class IndexDocValues implements Closeable { public Source getSource() throws IOException { return cache.load(this); } - - /** - * Returns a {@link SortedSource} instance for this {@link IndexDocValues} field - * instance like {@link #getSource()}. - *

- * This method will return null iff this {@link IndexDocValues} represent a - * {@link Source} instead of a {@link SortedSource}. - */ - public SortedSource getSortedSorted(Comparator comparator) - throws IOException { - return cache.loadSorted(this, comparator); - } - /** - * Returns a {@link SortedSource} instance using a default {@link BytesRef} - * comparator for this {@link IndexDocValues} field instance like - * {@link #getSource()}. - *

- * This method will return null iff this {@link IndexDocValues} represent a - * {@link Source} instead of a {@link SortedSource}. - */ - public SortedSource getSortedSorted() throws IOException { - return getSortedSorted(null); - } + public abstract Source getDirectSource() throws IOException; /** - * Loads and returns a {@link SortedSource} instance for this - * {@link IndexDocValues} field instance like {@link #load()}. - *

- * This method will return null iff this {@link IndexDocValues} represent a - * {@link Source} instead of a {@link SortedSource}. - */ - public SortedSource loadSorted(Comparator comparator) - throws IOException { - throw new UnsupportedOperationException(); - } - - /** - * Loads and returns a {@link SortedSource} instance using a default - * {@link BytesRef} comparator for this {@link IndexDocValues} field instance - * like {@link #load()}. - *

- * This method will return null iff this {@link IndexDocValues} represent a - * {@link Source} instead of a {@link SortedSource}. - */ - public SortedSource loadSorted() throws IOException { - return loadSorted(null); - } - - /** * Returns the {@link ValueType} of this {@link IndexDocValues} instance */ public abstract ValueType type(); @@ -242,6 +175,7 @@ public abstract class IndexDocValues implements Closeable { * Returns a {@link BytesRef} for the given document id or throws an * {@link UnsupportedOperationException} if this source doesn't support * byte[] values. + * @throws IOException * * @throws UnsupportedOperationException * if this source doesn't support byte[] values. @@ -257,14 +191,7 @@ public abstract class IndexDocValues implements Closeable { public int getValueCount() { throw new UnsupportedOperationException(); } - - /** - * Returns a {@link ValuesEnum} for this source. - */ - public ValuesEnum getEnum() throws IOException { - return getEnum(null); - } - + /** * Returns the {@link ValueType} of this source. * @@ -273,13 +200,6 @@ public abstract class IndexDocValues implements Closeable { public abstract ValueType type(); /** - * Returns a {@link ValuesEnum} for this source which uses the given - * {@link AttributeSource}. - */ - public abstract ValuesEnum getEnum(AttributeSource attrSource) - throws IOException; - - /** * Returns true iff this {@link Source} exposes an array via * {@link #getArray()} otherwise false. * @@ -298,110 +218,4 @@ public abstract class IndexDocValues implements Closeable { return null; } } - - /** - * {@link ValuesEnum} utility for {@link Source} implemenations. - * - */ - public abstract static class SourceEnum extends ValuesEnum { - protected final Source source; - protected final int numDocs; - protected int pos = -1; - - /** - * Creates a new {@link SourceEnum} - * - * @param attrs - * the {@link AttributeSource} for this enum - * @param type - * the enums {@link ValueType} - * @param source - * the source this enum operates on - * @param numDocs - * the number of documents within the source - */ - protected SourceEnum(AttributeSource attrs, ValueType type, Source source, - int numDocs) { - super(attrs, type); - this.source = source; - this.numDocs = numDocs; - } - - @Override - public void close() throws IOException { - } - - @Override - public int docID() { - return pos; - } - - @Override - public int nextDoc() throws IOException { - if (pos == NO_MORE_DOCS) - return NO_MORE_DOCS; - return advance(pos + 1); - } - } - - /** - * A sorted variant of {@link Source} for byte[] values per document. - *

- * Note: {@link ValuesEnum} obtained from a {@link SortedSource} will - * enumerate values in document order and not in sorted order. - */ - public static abstract class SortedSource extends Source { - - @Override - public BytesRef getBytes(int docID, BytesRef bytesRef) { - final int ord = ord(docID); - if (ord < 0) { - bytesRef.length = 0; - } else { - getByOrd(ord , bytesRef); - } - return bytesRef; - } - - /** - * Returns ord for specified docID. If this docID had not been added to the - * Writer, the ord is 0. Ord is dense, ie, starts at 0, then increments by 1 - * for the next (as defined by {@link Comparator} value. - */ - public abstract int ord(int docID); - - /** Returns value for specified ord. */ - public abstract BytesRef getByOrd(int ord, BytesRef bytesRef); - - - /** - * Finds the ordinal whose value is greater or equal to the given value. - * - * @return the given values ordinal if found or otherwise - * (-(ord)-1), defined as the ordinal of the first - * element that is greater than the given value. This guarantees - * that the return value will always be >= 0 if the given value - * is found. - * - */ - public final int getByValue(BytesRef value) { - return getByValue(value, new BytesRef()); - } - - /** - * Performs a lookup by value. - * - * @param value - * the value to look up - * @param tmpRef - * a temporary {@link BytesRef} instance used to compare internal - * values to the given value. Must not be null - * @return the given values ordinal if found or otherwise - * (-(ord)-1), defined as the ordinal of the first - * element that is greater than the given value. This guarantees - * that the return value will always be >= 0 if the given value - * is found. - */ - public abstract int getByValue(BytesRef value, BytesRef tmpRef); - } } diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java index b9041d0..9074161 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java @@ -2,9 +2,7 @@ package org.apache.lucene.index.values; import java.io.IOException; -import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedStraightBytesEnum; import org.apache.lucene.index.values.IndexDocValues.Source; -import org.apache.lucene.index.values.IndexDocValues.SourceEnum; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -71,41 +69,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - if (isFloat) { - return new SourceEnum(attrSource, type(), this, maxDocID + 1) { - - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - floatsRef.floats[intsRef.offset] = IndexDocValuesArray.this - .getFloat(target); - return pos = target; - } - }; - } else { - return new SourceEnum(attrSource, type(), this, maxDocID + 1) { - - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - intsRef.ints[intsRef.offset] = IndexDocValuesArray.this - .getInt(target); - return pos = target; - } - - }; - } - } - - abstract ValuesEnum getDirectEnum(AttributeSource attrSource, - IndexInput input, int maxDoc) throws IOException; - - @Override public final boolean hasArray() { return true; } @@ -137,19 +100,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FixedIntsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - - @Override - protected final long toLong(BytesRef bytesRef) { - return bytesRef.bytes[bytesRef.offset]; - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new ByteValues(input, numDocs); @@ -186,19 +136,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FixedIntsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - - @Override - protected final long toLong(BytesRef bytesRef) { - return bytesRef.asShort(); - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new ShortValues(input, numDocs); @@ -235,18 +172,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FixedIntsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - @Override - protected final long toLong(BytesRef bytesRef) { - return bytesRef.asInt(); - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new IntValues(input, numDocs); @@ -283,18 +208,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FixedIntsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - @Override - protected final long toLong(BytesRef bytesRef) { - return bytesRef.asLong(); - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new LongValues(input, numDocs); @@ -334,18 +247,6 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FloatsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - @Override - protected double toDouble(BytesRef bytesRef) { - return Float.intBitsToFloat(bytesRef.asInt()); - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new FloatValues(input, numDocs); @@ -384,78 +285,11 @@ abstract class IndexDocValuesArray extends Source { } @Override - ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input, - int maxDoc) throws IOException { - return new FloatsEnum(attrSource, input, type(), - bytesPerValue, maxDoc) { - @Override - protected double toDouble(BytesRef bytesRef) { - return Double.longBitsToDouble(bytesRef.asLong()); - } - }; - } - - @Override public IndexDocValuesArray newFromInput(IndexInput input, int numDocs) throws IOException { return new DoubleValues(input, numDocs); } - }; - - private abstract static class FixedIntsEnum extends - FixedStraightBytesEnum { - private final ValueType type; - private FixedIntsEnum(AttributeSource source, IndexInput dataIn, - ValueType type, int bytesPerValue, int maxDoc) throws IOException { - super(source, dataIn, bytesPerValue, maxDoc); - this.type = type; - - } - - @Override - public int advance(int target) throws IOException { - final int advance = super.advance(target); - if (advance != NO_MORE_DOCS) { - intsRef.ints[0] = toLong(this.bytesRef); - } - return advance; - } - - protected abstract long toLong(BytesRef bytesRef); - - @Override - public ValueType type() { - return type; - } - - } - - private abstract static class FloatsEnum extends FixedStraightBytesEnum { - - private final ValueType type; - FloatsEnum(AttributeSource source, IndexInput dataIn, ValueType type, int bytePerValue, int maxDoc) - throws IOException { - super(source, dataIn, bytePerValue, maxDoc); - this.type = type; - } - - @Override - public int advance(int target) throws IOException { - final int retVal = super.advance(target); - if (retVal != NO_MORE_DOCS) { - floatsRef.floats[floatsRef.offset] = toDouble(bytesRef); - } - return retVal; - } - - protected abstract double toDouble(BytesRef bytesRef); - - @Override - public ValueType type() { - return type; - } - - } + }; } diff --git a/lucene/src/java/org/apache/lucene/index/values/Ints.java b/lucene/src/java/org/apache/lucene/index/values/Ints.java index a8961a4..6d23c9f 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Ints.java +++ b/lucene/src/java/org/apache/lucene/index/values/Ints.java @@ -164,22 +164,6 @@ public final class Ints { } @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - final IndexInput input = cloneData(); - boolean success = false; - try { - final ValuesEnum valuesEnum = arrayTemplate.getDirectEnum(source, - input, maxDoc); - success = true; - return valuesEnum; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(input); - } - } - } - - @Override public ValueType type() { return type; } diff --git a/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java index e0b126d..6f2b63e 100644 --- a/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java @@ -19,7 +19,6 @@ package org.apache.lucene.index.values; import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ReaderUtil; @@ -57,13 +56,8 @@ public class MultiIndexDocValues extends IndexDocValues { } @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new MultiValuesEnum(docValuesIdx, starts); - } - - @Override public Source load() throws IOException { - return new MultiSource(docValuesIdx, starts); + return new MultiSource(docValuesIdx, starts, false); } public IndexDocValues reset(DocValuesIndex[] docValuesIdx) { @@ -86,11 +80,6 @@ public class MultiIndexDocValues extends IndexDocValues { } @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return emptySoruce.getEnum(attrSource); - } - - @Override public Source load() throws IOException { return emptySoruce; } @@ -99,84 +88,89 @@ public class MultiIndexDocValues extends IndexDocValues { public ValueType type() { return emptySoruce.type(); } - } - - private static class MultiValuesEnum extends ValuesEnum { - private DocValuesIndex[] docValuesIdx; - private final int maxDoc; - private int currentStart; - private int currentMax; - private int currentDoc = -1; - private ValuesEnum currentEnum; - private final int[] starts; - - public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts) - throws IOException { - super(docValuesIdx[0].docValues.type()); - this.docValuesIdx = docValuesIdx; - final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1]; - maxDoc = last.start + last.length; - final DocValuesIndex idx = docValuesIdx[0]; - currentEnum = idx.docValues.getEnum(this.attributes()); - currentEnum.copyFrom(this); - intsRef = currentEnum.intsRef; - currentMax = idx.length; - currentStart = 0; - this.starts = starts; - } - - @Override - public void close() throws IOException { - currentEnum.close(); - } - - @Override - public int advance(int target) throws IOException { - assert target > currentDoc : "target " + target - + " must be > than the current doc " + currentDoc; - int relativeDoc = target - currentStart; - do { - if (target >= maxDoc) {// we are beyond max doc - return currentDoc = NO_MORE_DOCS; - } - if (target >= currentMax) { - final int idx = ReaderUtil.subIndex(target, starts); - currentEnum.close(); - currentEnum = docValuesIdx[idx].docValues.getEnum(); - currentEnum.copyFrom(this); - currentStart = docValuesIdx[idx].start; - currentMax = currentStart + docValuesIdx[idx].length; - relativeDoc = target - currentStart; - } - target = currentMax; // make sure that we advance to the next enum if the current is exhausted - - } while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS); - return currentDoc = currentStart + relativeDoc; - } - @Override - public int docID() { - return currentDoc; - } @Override - public int nextDoc() throws IOException { - return advance(currentDoc + 1); + public Source getDirectSource() throws IOException { + return emptySoruce; } } +// private static class MultiValuesEnum extends ValuesEnum { +// private DocValuesIndex[] docValuesIdx; +// private final int maxDoc; +// private int currentStart; +// private int currentMax; +// private int currentDoc = -1; +// private ValuesEnum currentEnum; +// private final int[] starts; +// +// public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts) +// throws IOException { +// super(docValuesIdx[0].docValues.type()); +// this.docValuesIdx = docValuesIdx; +// final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1]; +// maxDoc = last.start + last.length; +// final DocValuesIndex idx = docValuesIdx[0]; +// currentEnum = idx.docValues.getEnum(this.attributes()); +// currentEnum.copyFrom(this); +// intsRef = currentEnum.intsRef; +// currentMax = idx.length; +// currentStart = 0; +// this.starts = starts; +// } +// +// @Override +// public void close() throws IOException { +// currentEnum.close(); +// } +// +// @Override +// public int seek(int target) throws IOException { +// int relativeDoc = target - currentStart; +// do { +// if (target >= maxDoc) {// we are beyond max doc +// return currentDoc = NO_MORE_DOCS; +// } +// if (target >= currentMax || target < currentStart) { +// final int idx = ReaderUtil.subIndex(target, starts); +// currentEnum.close(); +// currentEnum = docValuesIdx[idx].docValues.getEnum(); +// currentEnum.copyFrom(this); +// currentStart = docValuesIdx[idx].start; +// currentMax = currentStart + docValuesIdx[idx].length; +// relativeDoc = target - currentStart; +// } +// target = currentMax; // make sure that we advance to the next enum if the current is exhausted +// +// } while ((relativeDoc = currentEnum.seek(relativeDoc)) == NO_MORE_DOCS); +// return currentDoc = currentStart + relativeDoc; +// } +// +// @Override +// public int docID() { +// return currentDoc; +// } +// +// @Override +// public int nextDoc() throws IOException { +// return seek(currentDoc + 1); +// } +// } + private static class MultiSource extends Source { private int numDocs = 0; private int start = 0; private Source current; private final int[] starts; private final DocValuesIndex[] docValuesIdx; + private boolean direct; - public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts) { + public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts, boolean direct) { this.docValuesIdx = docValuesIdx; this.starts = starts; assert docValuesIdx.length != 0; - + this.direct = direct; } public long getInt(int docID) { @@ -193,7 +187,11 @@ public class MultiIndexDocValues extends IndexDocValues { + " for doc id: " + docID + " slices : " + Arrays.toString(starts); assert docValuesIdx[idx] != null; try { - current = docValuesIdx[idx].docValues.getSource(); + if (direct) { + current = docValuesIdx[idx].docValues.getDirectSource(); + } else { + current = docValuesIdx[idx].docValues.getSource(); + } } catch (IOException e) { throw new RuntimeException("load failed", e); // TODO how should we // handle this @@ -216,11 +214,6 @@ public class MultiIndexDocValues extends IndexDocValues { } @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - throw new UnsupportedOperationException(); // TODO - } - - @Override public ValueType type() { return docValuesIdx[0].docValues.type(); } @@ -252,11 +245,6 @@ public class MultiIndexDocValues extends IndexDocValues { } @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return ValuesEnum.emptyEnum(type); - } - - @Override public ValueType type() { return type; } @@ -266,4 +254,9 @@ public class MultiIndexDocValues extends IndexDocValues { public ValueType type() { return this.docValuesIdx[0].docValues.type(); } + + @Override + public Source getDirectSource() throws IOException { + return new MultiSource(docValuesIdx, starts, true); + } } diff --git a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java index b73d651..7fe2224 100644 --- a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java @@ -21,18 +21,15 @@ import java.io.IOException; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedBytesWriterBase; import org.apache.lucene.index.values.IndexDocValues.Source; -import org.apache.lucene.index.values.IndexDocValues.SourceEnum; import org.apache.lucene.index.values.IndexDocValuesArray.LongValues; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.PackedInts; /** @@ -51,7 +48,6 @@ class PackedIntValues { static class PackedIntsWriter extends FixedBytesWriterBase { - private LongsRef intsRef; private long minValue; private long maxValue; private boolean started; @@ -114,10 +110,10 @@ class PackedIntValues { } @Override - protected void mergeDoc(int docID) throws IOException { + protected void mergeDoc(int docID, int sourceDoc) throws IOException { assert docID > lastDocId : "docID: " + docID + " must be greater than the last added doc id: " + lastDocId; - add(docID, intsRef.get()); + add(docID, currentMergeSource.getInt(sourceDoc)); } private void writePackedInts(IndexOutput datOut, int docCount) throws IOException { @@ -139,12 +135,6 @@ class PackedIntValues { w.add(defaultValue); } w.finish(); - w.finish(); - } - - @Override - protected void setNextEnum(ValuesEnum valuesEnum) { - intsRef = valuesEnum.getInt(); } @Override @@ -215,30 +205,17 @@ class PackedIntValues { datIn.close(); } - @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - final IndexInput input = (IndexInput) datIn.clone(); - boolean success = false; - try { - final ValuesEnum inst; - if (values == null) { - inst = new PackedIntsEnumImpl(source, input); - } else { - inst = values.getDirectEnum(source, input, numDocs); - } - success = true; - return inst; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(input); - } - } - } @Override public ValueType type() { return ValueType.VAR_INTS; } + + + @Override + public Source getDirectSource() throws IOException { + return values != null ? new FixedStraightBytesImpl.DirectFixedSource((IndexInput) datIn.clone(), 8, ValueType.FIXED_INTS_64) : new DirectPackedIntsSource((IndexInput) datIn.clone()); + } } @@ -248,7 +225,6 @@ class PackedIntValues { private final PackedInts.Reader values; public PackedIntsSource(IndexInput dataIn) throws IOException { - minValue = dataIn.readLong(); defaultValue = dataIn.readLong(); values = PackedInts.getReader(dataIn); @@ -265,70 +241,36 @@ class PackedIntValues { } @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return new SourceEnum(attrSource, type(), this, values.size()) { - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) - return pos = NO_MORE_DOCS; - intsRef.ints[intsRef.offset] = source.getInt(target); - return pos = target; - } - }; - } - - @Override public ValueType type() { return ValueType.VAR_INTS; } } - private static final class PackedIntsEnumImpl extends ValuesEnum { - private final PackedInts.ReaderIterator ints; + private static final class DirectPackedIntsSource extends Source { + private final PackedInts.SeekableReaderIterator ints; private long minValue; - private final IndexInput dataIn; private final long defaultValue; - private final int maxDoc; - private int pos = -1; - private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn) + private DirectPackedIntsSource(IndexInput dataIn) throws IOException { - super(source, ValueType.VAR_INTS); - intsRef.offset = 0; - this.dataIn = dataIn; minValue = dataIn.readLong(); defaultValue = dataIn.readLong(); - this.ints = PackedInts.getReaderIterator(dataIn); - maxDoc = ints.size(); - } - - @Override - public void close() throws IOException { - ints.close(); - dataIn.close(); + this.ints = PackedInts.getSeekableReaderIterator(dataIn); } @Override - public int advance(int target) throws IOException { - if (target >= maxDoc) { - return pos = NO_MORE_DOCS; + public long getInt(int docID) { + try { + final long val = ints.seek(docID); + return val == defaultValue ? 0 : minValue + val; + } catch (IOException e) { + throw new RuntimeException(e); } - final long val = ints.advance(target); - intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val; - return pos = target; } @Override - public int docID() { - return pos; - } - - @Override - public int nextDoc() throws IOException { - if (pos >= maxDoc) { - return pos = NO_MORE_DOCS; - } - return advance(pos + 1); + public ValueType type() { + return ValueType.VAR_INTS; } } diff --git a/lucene/src/java/org/apache/lucene/index/values/SourceCache.java b/lucene/src/java/org/apache/lucene/index/values/SourceCache.java index 7080006..afa0c80 100644 --- a/lucene/src/java/org/apache/lucene/index/values/SourceCache.java +++ b/lucene/src/java/org/apache/lucene/index/values/SourceCache.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Comparator; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -63,16 +62,6 @@ public abstract class SourceCache { public abstract Source load(IndexDocValues values) throws IOException; /** - * Atomically loads a {@link SortedSource} into the cache from the given - * {@link IndexDocValues} and returns it iff no other {@link SortedSource} has - * already been cached. Otherwise the cached source is returned. - *

- * This method will not return null - */ - public abstract SortedSource loadSorted(IndexDocValues values, - Comparator comp) throws IOException; - - /** * Atomically invalidates the cached {@link Source} and {@link SortedSource} * instances if any and empties the cache. */ @@ -94,7 +83,6 @@ public abstract class SourceCache { */ public static final class DirectSourceCache extends SourceCache { private Source ref; - private SortedSource sortedRef; public synchronized Source load(IndexDocValues values) throws IOException { if (ref == null) { @@ -103,17 +91,8 @@ public abstract class SourceCache { return ref; } - public synchronized SortedSource loadSorted(IndexDocValues values, - Comparator comp) throws IOException { - if (sortedRef == null) { - sortedRef = values.loadSorted(comp); - } - return sortedRef; - } - public synchronized void invalidate(IndexDocValues values) { ref = null; - sortedRef = null; } } diff --git a/lucene/src/java/org/apache/lucene/index/values/ValueType.java b/lucene/src/java/org/apache/lucene/index/values/ValueType.java index 974d496..f843453 100644 --- a/lucene/src/java/org/apache/lucene/index/values/ValueType.java +++ b/lucene/src/java/org/apache/lucene/index/values/ValueType.java @@ -18,7 +18,6 @@ package org.apache.lucene.index.values; */ import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.packed.PackedInts; @@ -167,26 +166,6 @@ public enum ValueType { BYTES_FIXED_DEREF, /** - * A fixed length pre-sorted byte[] variant. Fields with this type only - * store distinct byte values and store an additional offset pointer per - * document to dereference the shared byte[]. The stored - * byte[] is presorted, by default by unsigned byte order, - * and allows access via document id, ordinal and by-value. - * Use this type if your documents may share the same byte[]. - *

- * NOTE: Fields of this type will not store values for documents without and - * explicitly provided value. If a documents value is accessed while no - * explicit value is stored the returned {@link BytesRef} will be a 0-length - * reference. In turn, {@link ValuesEnum} instances will skip over documents - * without an explicit value assigned. Custom default values must be assigned - * explicitly. - *

- * - * @see SortedSource - */ - BYTES_FIXED_SORTED, - - /** * Variable length straight stored byte[] variant. All bytes are * stored sequentially for compactness. Usage of this type via the * disk-resident API might yield performance degradation since no additional @@ -216,21 +195,4 @@ public enum ValueType { *

*/ BYTES_VAR_DEREF, - - /** - * A variable length pre-sorted byte[] variant. Just like - * {@link #BYTES_FIXED_SORTED}, but allowing each - * document's value to be a different length. - *

- * NOTE: Fields of this type will not store values for documents without and - * explicitly provided value. If a documents value is accessed while no - * explicit value is stored the returned {@link BytesRef} will be a 0-length - * reference. In turn, {@link ValuesEnum} instances will skip over documents - * without an explicit value assigned. Custom default values must be assigned - * explicitly. - *

- * - * @see SortedSource - */ - BYTES_VAR_SORTED } diff --git a/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java b/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java deleted file mode 100644 index 3c48476..0000000 --- a/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.apache.lucene.index.values; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; - -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FloatsRef; -import org.apache.lucene.util.LongsRef; - -/** - * {@link ValuesEnum} is a {@link DocIdSetIterator} iterating byte[] - * , long and double stored per document. Depending on the - * enum's {@link ValueType} ({@link #type()}) the enum might skip over documents that - * have no value stored. Types like {@link ValueType#BYTES_VAR_STRAIGHT} might not - * skip over documents even if there is no value associated with a document. The - * value for document without values again depends on the types implementation - * although a reference for a {@link ValueType} returned from a accessor method - * {@link #getFloat()}, {@link #getInt()} or {@link #bytes()} will never be - * null even if a document has no value. - *

- * Note: Only the reference for the enum's type are initialized to non - * null ie. {@link #getInt()} will always return null - * if the enum's Type is {@link ValueType#FLOAT_32}. - * - * @lucene.experimental - */ -public abstract class ValuesEnum extends DocIdSetIterator { - private AttributeSource source; - private final ValueType enumType; - protected BytesRef bytesRef = new BytesRef(1); - protected FloatsRef floatsRef = new FloatsRef(1); - protected LongsRef intsRef = new LongsRef(1); - - /** - * Creates a new {@link ValuesEnum} for the given type. The - * {@link AttributeSource} for this enum is set to null - */ - protected ValuesEnum(ValueType enumType) { - this(null, enumType); - } - - /** - * Creates a new {@link ValuesEnum} for the given type. - */ - protected ValuesEnum(AttributeSource source, ValueType enumType) { - this.source = source; - this.enumType = enumType; - } - - /** - * Returns the type of this enum - */ - public ValueType type() { - return enumType; - } - - /** - * Returns a {@link BytesRef} or null if this enum doesn't - * enumerate byte[] values - */ - public BytesRef bytes() { - return bytesRef; - } - - /** - * Returns a {@link FloatsRef} or null if this enum doesn't - * enumerate floating point values - */ - public FloatsRef getFloat() { - return floatsRef; - } - - /** - * Returns a {@link LongsRef} or null if this enum doesn't - * enumerate integer values. - */ - public LongsRef getInt() { - return intsRef; - } - - /** - * Copies the internal state from the given enum - */ - protected void copyFrom(ValuesEnum valuesEnum) { - intsRef = valuesEnum.intsRef; - floatsRef = valuesEnum.floatsRef; - bytesRef = valuesEnum.bytesRef; - source = valuesEnum.source; - } - - /** - * Returns the {@link AttributeSource} associated with this enum. - *

- * Note: this method might create a new AttribueSource if no - * {@link AttributeSource} has been provided during enum creation. - */ - public AttributeSource attributes() { - if (source == null) { - source = new AttributeSource(); - } - return source; - } - - /** - * Closes the enum - * - * @throws IOException - * if an {@link IOException} occurs - */ - public abstract void close() throws IOException; - - /** - * Returns an empty {@link ValuesEnum} for the given {@link ValueType}. - */ - public static ValuesEnum emptyEnum(ValueType type) { - return new ValuesEnum(type) { - @Override - public int nextDoc() throws IOException { - return NO_MORE_DOCS; - } - - @Override - public int docID() { - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) throws IOException { - return NO_MORE_DOCS; - } - - @Override - public void close() throws IOException { - - } - }; - } - -} diff --git a/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java index 2ea2ef0..97f0a5d 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java @@ -21,15 +21,15 @@ import java.io.IOException; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase; -import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase; import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; +import org.apache.lucene.index.values.DirectSource; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; // Stores variable-length byte[] by deref, ie when two docs // have the same value, they store only 1 byte[] and both @@ -57,6 +57,7 @@ class VarDerefBytesImpl { public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); + size = 0; } @Override @@ -68,22 +69,23 @@ class VarDerefBytesImpl { // some last docs that we didn't see @Override public void finishInternal(int docCount) throws IOException { + fillDefault(docCount); final int size = hash.size(); - final long[] addresses = new long[size+1]; + final long[] addresses = new long[size]; final IndexOutput datOut = getOrCreateDataOut(); - int addr = 1; + int addr = 0; final BytesRef bytesRef = new BytesRef(); for (int i = 0; i < size; i++) { hash.get(i, bytesRef); - addresses[i+1] = addr; + addresses[i] = addr; addr += writePrefixLength(datOut, bytesRef) + bytesRef.length; datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } final IndexOutput idxOut = getOrCreateIndexOut(); // write the max address to read directly on source load - idxOut.writeLong(addr - 1); - writeIndex(idxOut, docCount, addresses[size], addresses, docToEntry); + idxOut.writeLong(addr); + writeIndex(idxOut, docCount, addresses[addresses.length-1], addresses, docToEntry); } } @@ -108,48 +110,48 @@ class VarDerefBytesImpl { @Override public BytesRef getBytes(int docID, BytesRef bytesRef) { - long address = addresses.get(docID); - bytesRef.length = 0; - return address == 0 ? bytesRef : data.fillSliceWithPrefix(bytesRef, - --address); + return data.fillSliceWithPrefix(bytesRef, + addresses.get(docID)); } } @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new VarDerefBytesEnum(source, cloneData(), cloneIndex()); + public ValueType type() { + return ValueType.BYTES_VAR_DEREF; } - final static class VarDerefBytesEnum extends DerefBytesEnumBase { - - public VarDerefBytesEnum(AttributeSource source, IndexInput datIn, - IndexInput idxIn) throws IOException { - super(source, datIn, idxIn, -1, ValueType.BYTES_VAR_DEREF); - } + @Override + public org.apache.lucene.index.values.IndexDocValues.Source getDirectSource() + throws IOException { + return new DirectDerefVarSource(cloneData(), cloneIndex(), type()); + } + } + + public final static class DirectDerefVarSource extends DirectSource { - @Override - protected void fill(long address, BytesRef ref) throws IOException { - datIn.seek(fp + --address); - final byte sizeByte = datIn.readByte(); - final int size; - if ((sizeByte & 128) == 0) { - // length is 1 byte - size = sizeByte; - } else { - size = ((sizeByte & 0x7f) << 8) | ((datIn.readByte() & 0xff)); - } - if (ref.bytes.length < size) { - ref.grow(size); - } - ref.length = size; - ref.offset = 0; - datIn.readBytes(ref.bytes, 0, size); - } + private PackedInts.SeekableReaderIterator index; + + DirectDerefVarSource(IndexInput data, IndexInput index, ValueType type) + throws IOException { + super(data, type); + this.index = PackedInts.getSeekableReaderIterator(index); } @Override - public ValueType type() { - return ValueType.BYTES_VAR_DEREF; + protected void offsetAndSize(int docID, OffsetAndSize offsetAndSize) + throws IOException { + offsetAndSize.offset = index.seek(docID); + data.seek(baseOffset + offsetAndSize.offset); + final byte sizeByte = data.readByte(); + offsetAndSize.offset++; + if ((sizeByte & 128) == 0) { + // length is 1 byte + offsetAndSize.size = sizeByte; + } else { + offsetAndSize.size = ((sizeByte & 0x7f) << 8) | ((data.readByte() & 0xff)); + offsetAndSize.offset++; + } + } } } diff --git a/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java deleted file mode 100644 index 36caf0b..0000000 --- a/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java +++ /dev/null @@ -1,247 +0,0 @@ -package org.apache.lucene.index.values; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase; -import org.apache.lucene.index.values.Bytes.BytesReaderBase; -import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Counter; -import org.apache.lucene.util.packed.PackedInts; - -// Stores variable-length byte[] by deref, ie when two docs -// have the same value, they store only 1 byte[] and both -// docs reference that single source - -/** - * @lucene.experimental - */ -class VarSortedBytesImpl { - - static final String CODEC_NAME = "VarDerefBytes"; - static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; - - final static class Writer extends DerefBytesWriterBase { - private final Comparator comp; - - public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); - this.comp = comp; - } - - @Override - protected void checkSize(BytesRef bytes) { - // allow var bytes sizes - } - - // Important that we get docCount, in case there were - // some last docs that we didn't see - @Override - public void finishInternal(int docCount) throws IOException { - final int count = hash.size(); - final IndexOutput datOut = getOrCreateDataOut(); - long offset = 0; - long lastOffset = 0; - final int[] index = new int[count+1]; - final long[] offsets = new long[count]; - final int[] sortedEntries = hash.sort(comp); - // first dump bytes data, recording index & offset as - // we go - for (int i = 0; i < count; i++) { - final int e = sortedEntries[i]; - offsets[i] = offset; - index[e+1] = 1 + i; - - final BytesRef bytes = hash.get(e, new BytesRef()); - // TODO: we could prefix code... - datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length); - lastOffset = offset; - offset += bytes.length; - } - - final IndexOutput idxOut = getOrCreateIndexOut(); - // total bytes of data - idxOut.writeLong(offset); - // write index -- first doc -> 1+ord - writeIndex(idxOut, docCount, count, index, docToEntry); - // next ord (0-based) -> offset - PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count, - PackedInts.bitsRequired(lastOffset)); - for (int i = 0; i < count; i++) { - offsetWriter.add(offsets[i]); - } - offsetWriter.finish(); - } - } - - public static class Reader extends BytesReaderBase { - - private final Comparator defaultComp; - - Reader(Directory dir, String id, int maxDoc, - Comparator comparator, IOContext context) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true, context); - this.defaultComp = comparator; - } - - @Override - public org.apache.lucene.index.values.IndexDocValues.Source load() - throws IOException { - return loadSorted(defaultComp); - } - - @Override - public SortedSource loadSorted(Comparator comp) - throws IOException { - IndexInput indexIn = cloneIndex(); - return new Source(cloneData(), indexIn, comp, indexIn.readLong()); - } - - private static class Source extends BytesSortedSourceBase { - private final PackedInts.Reader ordToOffsetIndex; // 0-based - private final long totBytes; - private final int valueCount; - - public Source(IndexInput datIn, IndexInput idxIn, - Comparator comp, long dataLength) throws IOException { - super(datIn, idxIn, comp, dataLength, ValueType.BYTES_VAR_SORTED); - totBytes = dataLength; - ordToOffsetIndex = PackedInts.getReader(idxIn); - valueCount = ordToOffsetIndex.size(); - closeIndexInput(); - } - - @Override - public int getByValue(BytesRef bytes, BytesRef tmpRef) { - return binarySearch(bytes, tmpRef, 0, valueCount - 1); - } - - @Override - public int getValueCount() { - return valueCount; - } - - // ord is 0-based - @Override - protected BytesRef deref(int ord, BytesRef bytesRef) { - final long nextOffset; - if (ord == valueCount - 1) { - nextOffset = totBytes; - } else { - nextOffset = ordToOffsetIndex.get(1 + ord); - } - final long offset = ordToOffsetIndex.get(ord); - data.fillSlice(bytesRef, offset, (int) (nextOffset - offset)); - return bytesRef; - } - } - - @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new VarSortedBytesEnum(source, cloneData(), cloneIndex()); - } - - private static class VarSortedBytesEnum extends ValuesEnum { - private PackedInts.Reader docToOrdIndex; - private PackedInts.Reader ordToOffsetIndex; - private IndexInput idxIn; - private IndexInput datIn; - private int valueCount; - private long totBytes; - private int docCount; - private int pos = -1; - private final long fp; - - protected VarSortedBytesEnum(AttributeSource source, IndexInput datIn, - IndexInput idxIn) throws IOException { - super(source, ValueType.BYTES_VAR_SORTED); - totBytes = idxIn.readLong(); - // keep that in memory to prevent lots of disk seeks - docToOrdIndex = PackedInts.getReader(idxIn); - ordToOffsetIndex = PackedInts.getReader(idxIn); - valueCount = ordToOffsetIndex.size(); - docCount = docToOrdIndex.size(); - fp = datIn.getFilePointer(); - this.idxIn = idxIn; - this.datIn = datIn; - } - - @Override - public void close() throws IOException { - idxIn.close(); - datIn.close(); - } - - @Override - public int advance(int target) throws IOException { - if (target >= docCount) { - return pos = NO_MORE_DOCS; - } - int ord; - while ((ord = (int) docToOrdIndex.get(target)) == 0) { - if (++target >= docCount) { - return pos = NO_MORE_DOCS; - } - } - final long offset = ordToOffsetIndex.get(--ord); - final long nextOffset; - if (ord == valueCount - 1) { - nextOffset = totBytes; - } else { - nextOffset = ordToOffsetIndex.get(1 + ord); - } - final int length = (int) (nextOffset - offset); - datIn.seek(fp + offset); - if (bytesRef.bytes.length < length) - bytesRef.grow(length); - datIn.readBytes(bytesRef.bytes, 0, length); - bytesRef.length = length; - bytesRef.offset = 0; - return pos = target; - } - - @Override - public int docID() { - return pos; - } - - @Override - public int nextDoc() throws IOException { - if (pos >= docCount) { - return pos = NO_MORE_DOCS; - } - return advance(pos + 1); - } - } - - @Override - public ValueType type() { - return ValueType.BYTES_VAR_SORTED; - } - } -} diff --git a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java index 0d1282a..0435489 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java @@ -22,12 +22,12 @@ import java.io.IOException; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase; +import org.apache.lucene.index.values.DirectSource; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -66,7 +66,7 @@ class VarStraightBytesImpl { } // Fills up to but not including this docID - private void fill(final int docID) { + private void fill(final int docID, final long nextAddress) { if (docID >= docToAddress.length) { int oldSize = docToAddress.length; docToAddress = ArrayUtil.grow(docToAddress, 1 + docID); @@ -74,7 +74,7 @@ class VarStraightBytesImpl { * RamUsageEstimator.NUM_BYTES_INT); } for (int i = lastDocID + 1; i < docID; i++) { - docToAddress[i] = address; + docToAddress[i] = nextAddress; } } @@ -84,7 +84,7 @@ class VarStraightBytesImpl { if (bytes.length == 0) { return; // default } - fill(docID); + fill(docID, address); docToAddress[docID] = address; pool.copy(bytes); address += bytes.length; @@ -105,7 +105,7 @@ class VarStraightBytesImpl { return; } if (lastDocID+1 < state.docBase) { - fill(state.docBase); + fill(state.docBase, address); lastDocID = state.docBase-1; } final long numDataBytes; @@ -147,13 +147,14 @@ class VarStraightBytesImpl { } @Override - protected void mergeDoc(int docID) throws IOException { + protected void mergeDoc(int docID, int sourceDoc) throws IOException { assert merge; assert lastDocID < docID; + currentMergeSource.getBytes(sourceDoc, bytesRef); if (bytesRef.length == 0) { return; // default } - fill(docID); + fill(docID, address); datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); docToAddress[docID] = address; address += bytesRef.length; @@ -186,20 +187,21 @@ class VarStraightBytesImpl { try { if (lastDocID == -1) { idxOut.writeVLong(0); - final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, + final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1, PackedInts.bitsRequired(0)); - for (int i = 0; i < docCount; i++) { + for (int i = 0; i < docCount+1; i++) { w.add(0); } w.finish(); } else { - fill(docCount); + fill(docCount, address); idxOut.writeVLong(address); - final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, + final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1, PackedInts.bitsRequired(address)); for (int i = 0; i < docCount; i++) { w.add(docToAddress[i]); } + w.add(address); w.finish(); } success = true; @@ -234,7 +236,6 @@ class VarStraightBytesImpl { } private class Source extends DerefBytesSourceBase { - public Source(IndexInput datIn, IndexInput idxIn) throws IOException { super(datIn, idxIn, idxIn.readVLong(), ValueType.BYTES_VAR_STRAIGHT); } @@ -242,93 +243,40 @@ class VarStraightBytesImpl { @Override public BytesRef getBytes(int docID, BytesRef bytesRef) { final long address = addresses.get(docID); - final int length = docID == maxDoc - 1 ? (int) (totalLengthInBytes - address) - : (int) (addresses.get(1 + docID) - address); - return data.fillSlice(bytesRef, address, length); - } - - @Override - public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { - return new SourceEnum(attrSource, type(), this, maxDoc()) { - @Override - public int advance(int target) throws IOException { - if (target >= numDocs) { - return pos = NO_MORE_DOCS; - } - source.getBytes(target, bytesRef); - return pos = target; - } - }; + return data.fillSlice(bytesRef, address, + (int) (addresses.get(docID + 1) - address)); } } @Override - public ValuesEnum getEnum(AttributeSource source) throws IOException { - return new VarStraightBytesEnum(source, cloneData(), cloneIndex()); + public ValueType type() { + return ValueType.BYTES_VAR_STRAIGHT; } - private class VarStraightBytesEnum extends ValuesEnum { - private final PackedInts.ReaderIterator addresses; - private final IndexInput datIn; - private final IndexInput idxIn; - private final long fp; - private final long totBytes; - private int pos = -1; - private long nextAddress; - - protected VarStraightBytesEnum(AttributeSource source, IndexInput datIn, - IndexInput idxIn) throws IOException { - super(source, ValueType.BYTES_VAR_STRAIGHT); - totBytes = idxIn.readVLong(); - fp = datIn.getFilePointer(); - addresses = PackedInts.getReaderIterator(idxIn); - this.datIn = datIn; - this.idxIn = idxIn; - nextAddress = addresses.next(); - } - - @Override - public void close() throws IOException { - datIn.close(); - idxIn.close(); - } - - @Override - public int advance(final int target) throws IOException { - if (target >= maxDoc) { - return pos = NO_MORE_DOCS; - } - final long addr = pos+1 == target ? nextAddress : addresses.advance(target); - if (addr == totBytes) { // empty values at the end - bytesRef.length = 0; - bytesRef.offset = 0; - return pos = target; - } - datIn.seek(fp + addr); - final int size = (int) (target == maxDoc - 1 ? totBytes - addr - : (nextAddress = addresses.next()) - addr); - if (bytesRef.bytes.length < size) { - bytesRef.grow(size); - } - bytesRef.length = size; - datIn.readBytes(bytesRef.bytes, 0, size); - return pos = target; - } + @Override + public org.apache.lucene.index.values.IndexDocValues.Source getDirectSource() + throws IOException { + IndexInput cloneIndex = cloneIndex(); + cloneIndex.readVLong(); + return new DirectStraightVarSource(cloneData(), cloneIndex, type()); + } + } + + public final static class DirectStraightVarSource extends DirectSource { - @Override - public int docID() { - return pos; - } + private PackedInts.SeekableReaderIterator index; - @Override - public int nextDoc() throws IOException { - return advance(pos + 1); - } + DirectStraightVarSource(IndexInput data, IndexInput index, ValueType type) + throws IOException { + super(data, type); + this.index = PackedInts.getSeekableReaderIterator(index); } @Override - public ValueType type() { - return ValueType.BYTES_VAR_STRAIGHT; + protected void offsetAndSize(int docID, OffsetAndSize offsetAndSize) + throws IOException { + offsetAndSize.offset = index.seek(docID); + offsetAndSize.size = (int) (index.next() - offsetAndSize.offset); } } } diff --git a/lucene/src/java/org/apache/lucene/index/values/Writer.java b/lucene/src/java/org/apache/lucene/index/values/Writer.java index db22b12..529b57a 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Writer.java +++ b/lucene/src/java/org/apache/lucene/index/values/Writer.java @@ -17,9 +17,9 @@ package org.apache.lucene.index.values; * limitations under the License. */ import java.io.IOException; -import java.util.Comparator; import org.apache.lucene.index.codecs.DocValuesConsumer; +import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Bits; @@ -39,7 +39,7 @@ import org.apache.lucene.util.Counter; * @lucene.experimental */ public abstract class Writer extends DocValuesConsumer { - + protected Source currentMergeSource; /** * Creates a new {@link Writer}. * @@ -101,7 +101,7 @@ public abstract class Writer extends DocValuesConsumer { /** * Records a value from the given document id. The methods implementation * obtains the value for the document id from the last {@link ValuesEnum} - * set to {@link #setNextEnum(ValuesEnum)}. + * set to {@link #setNextMergeSource(Source)}. *

* This method is used during merging to provide implementation agnostic * default merge implementation. @@ -114,16 +114,18 @@ public abstract class Writer extends DocValuesConsumer { * the {@link Writer} implementation. The given document ID must always be * greater than the previous ID or 0 if called the first time. */ - protected abstract void mergeDoc(int docID) throws IOException; + protected abstract void mergeDoc(int docID, int sourceDoc) throws IOException; /** * Sets the next {@link ValuesEnum} to consume values from on calls to * {@link #mergeDoc(int)} * - * @param valuesEnum + * @param mergeSource * the next {@link ValuesEnum}, this must not be null */ - protected abstract void setNextEnum(ValuesEnum valuesEnum); + protected void setNextMergeSource(Source mergeSource) { + currentMergeSource = mergeSource; + } /** * Finish writing and close any files and resources used by this Writer. @@ -141,34 +143,20 @@ public abstract class Writer extends DocValuesConsumer { // simply override this and decide if they want to merge // segments using this generic implementation or if a bulk merge is possible // / feasible. - final ValuesEnum valEnum = state.reader.getEnum(); - assert valEnum != null; - try { - setNextEnum(valEnum); // set the current enum we are working on - the - // impl. will get the correct reference for the type - // it supports - int docID = state.docBase; - final Bits liveDocs = state.liveDocs; - final int docCount = state.docCount; - int currentDocId; - if ((currentDocId = valEnum.advance(0)) != ValuesEnum.NO_MORE_DOCS) { - for (int i = 0; i < docCount; i++) { - if (liveDocs == null || liveDocs.get(i)) { - if (currentDocId < i) { - if ((currentDocId = valEnum.advance(i)) == ValuesEnum.NO_MORE_DOCS) { - break; // advance can jump over default values - } - } - if (currentDocId == i) { // we are on the doc to merge - mergeDoc(docID); - } - ++docID; - } - } + final Source source = state.reader.getDirectSource(); + assert source != null; + setNextMergeSource(source); // set the current enum we are working on - the + // impl. will get the correct reference for the type + // it supports + int docID = state.docBase; + final Bits liveDocs = state.liveDocs; + final int docCount = state.docCount; + for (int i = 0; i < docCount; i++) { + if (liveDocs == null || liveDocs.get(i)) { + mergeDoc(docID++, i); } - } finally { - valEnum.close(); } + } /** @@ -193,10 +181,7 @@ public abstract class Writer extends DocValuesConsumer { * @throws IOException */ public static Writer create(ValueType type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context) throws IOException { - if (comp == null) { - comp = BytesRef.getUTF8SortedAsUnicodeComparator(); - } + Counter bytesUsed, IOContext context) throws IOException { switch (type) { case FIXED_INTS_16: case FIXED_INTS_32: @@ -209,24 +194,17 @@ public abstract class Writer extends DocValuesConsumer { case FLOAT_64: return Floats.getWriter(directory, id, 8, bytesUsed, context); case BYTES_FIXED_STRAIGHT: - return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, true, + return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, bytesUsed, context); case BYTES_FIXED_DEREF: - return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, true, - bytesUsed, context); - case BYTES_FIXED_SORTED: - return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, true, + return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, bytesUsed, context); case BYTES_VAR_STRAIGHT: - return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, false, + return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, bytesUsed, context); case BYTES_VAR_DEREF: - return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, false, - bytesUsed, context); - case BYTES_VAR_SORTED: - return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false, + return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, bytesUsed, context); - default: throw new IllegalArgumentException("Unknown Values: " + type); } diff --git a/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java b/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java index 589c8bc..35849c5 100644 --- a/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java +++ b/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java @@ -83,6 +83,17 @@ public class PackedInts { * @return the value at the given position * @throws IOException if reading the value throws an IOException*/ long advance(int ord) throws IOException; + + } + + /** + * Seekable enum interface, to decode previously saved PackedInts. + */ + public static interface SeekableReaderIterator extends ReaderIterator { + /** Seeks to the given ordinal and returns its value. + * @return the value at the given position + * @throws IOException if reading the value throws an IOException*/ + long seek(int ord) throws IOException; } /** @@ -195,6 +206,17 @@ public class PackedInts { * @lucene.internal */ public static ReaderIterator getReaderIterator(IndexInput in) throws IOException { + return getSeekableReaderIterator(in); + } + + /** + * Retrieve PackedInts as a {@link SeekableReaderIterator} + * @param in positioned at the beginning of a stored packed int structure. + * @return an iterator to access the values + * @throws IOException if the structure could not be retrieved. + * @lucene.internal + */ + public static SeekableReaderIterator getSeekableReaderIterator(IndexInput in) throws IOException { CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; diff --git a/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java b/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java index 90c67dc..5f5ba8a 100644 --- a/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java +++ b/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java @@ -21,13 +21,15 @@ import org.apache.lucene.store.IndexInput; import java.io.IOException; -final class PackedReaderIterator implements PackedInts.ReaderIterator { +final class PackedReaderIterator implements PackedInts.SeekableReaderIterator { private long pending; private int pendingBitsLeft; private final IndexInput in; private final int bitsPerValue; private final int valueCount; private int position = -1; + private long currentValue; + private final long startPointer; // masks[n-1] masks for bottom n bits private final long[] masks; @@ -39,6 +41,7 @@ final class PackedReaderIterator implements PackedInts.ReaderIterator { this.bitsPerValue = bitsPerValue; this.in = in; + startPointer = in.getFilePointer(); masks = new long[bitsPerValue]; long v = 1; @@ -76,7 +79,7 @@ final class PackedReaderIterator implements PackedInts.ReaderIterator { } ++position; - return result; + return currentValue = result; } public void close() throws IOException { @@ -106,6 +109,24 @@ final class PackedReaderIterator implements PackedInts.ReaderIterator { pendingBitsLeft = 64 - (int)(skip % 64); } position = ord-1; - return next(); + return currentValue = next(); + } + + public long seek(final int ord) throws IOException { + assert ord < valueCount : "ord must be less than valueCount"; + if (ord < position) { + pendingBitsLeft = 0; + final long bitsToSkip = (((long) bitsPerValue) * (long) ord); + final long skip = bitsToSkip - pendingBitsLeft; + final long closestByte = (skip >> 6) << 3; + in.seek(startPointer + closestByte); + pending = in.readLong(); + pendingBitsLeft = 64 - (int) (skip % 64); + position = ord - 1; + return currentValue = next(); + } else if (ord == position) { + return currentValue; + } + return advance(ord); } } diff --git a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java index b02ee56..7d61ab6 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java @@ -176,7 +176,6 @@ public class RandomIndexWriter implements Closeable { IndexDocValuesField docValuesField = new IndexDocValuesField(name); switch (type) { case BYTES_FIXED_DEREF: - case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength); BytesRef fixedRef = new BytesRef(randomUnicodeString); @@ -189,7 +188,6 @@ public class RandomIndexWriter implements Closeable { docValuesField.setBytes(fixedRef, type); break; case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200)); docValuesField.setBytes(ref, type); diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java index 35c1586..656e64a 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java @@ -18,15 +18,11 @@ package org.apache.lucene.index.values; */ import java.io.IOException; -import java.util.Comparator; -import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; -import org.apache.lucene.util.FloatsRef; -import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; @@ -46,22 +42,14 @@ public class TestDocValues extends LuceneTestCase { runTestBytes(Bytes.Mode.DEREF, false); } - public void testBytesSorted() throws IOException { - runTestBytes(Bytes.Mode.SORTED, true); - runTestBytes(Bytes.Mode.SORTED, false); - } - public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize) throws IOException { final BytesRef bytesRef = new BytesRef(); - final Comparator comp = mode == Bytes.Mode.SORTED ? BytesRef - .getUTF8SortedAsUnicodeComparator() : null; - Directory dir = newDirectory(); final Counter trackBytes = Counter.newCounter(); - Writer w = Bytes.getWriter(dir, "test", mode, comp, fixedSize, trackBytes, newIOContext(random)); + Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, trackBytes, newIOContext(random)); int maxDoc = 220; final String[] values = new String[maxDoc]; final int fixedLength = 1 + atLeast(50); @@ -81,90 +69,20 @@ public class TestDocValues extends LuceneTestCase { w.finish(maxDoc); assertEquals(0, trackBytes.get()); - IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, comp, newIOContext(random)); - for (int iter = 0; iter < 2; iter++) { - ValuesEnum bytesEnum = getEnum(r); - assertNotNull("enum is null", bytesEnum); - BytesRef ref = bytesEnum.bytes(); - - for (int i = 0; i < 2; i++) { - final int idx = 2 * i; - assertEquals("doc: " + idx, idx, bytesEnum.advance(idx)); - String utf8String = ref.utf8ToString(); - assertEquals("doc: " + idx + " lenLeft: " + values[idx].length() - + " lenRight: " + utf8String.length(), values[idx], utf8String); - } - assertEquals(ValuesEnum.NO_MORE_DOCS, bytesEnum.advance(maxDoc)); - assertEquals(ValuesEnum.NO_MORE_DOCS, bytesEnum.advance(maxDoc + 1)); - - bytesEnum.close(); - } + IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, newIOContext(random)); // Verify we can load source twice: for (int iter = 0; iter < 2; iter++) { Source s; - IndexDocValues.SortedSource ss; - if (mode == Bytes.Mode.SORTED) { - // default is unicode so we can simply pass null here - s = ss = getSortedSource(r, random.nextBoolean() ? comp : null); - } else { s = getSource(r); - ss = null; - } for (int i = 0; i < 100; i++) { final int idx = 2 * i; assertNotNull("doc " + idx + "; value=" + values[idx], s.getBytes(idx, bytesRef)); assertEquals("doc " + idx, values[idx], s.getBytes(idx, bytesRef) .utf8ToString()); - if (ss != null) { - assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx), - bytesRef).utf8ToString()); - int ord = ss - .getByValue(new BytesRef(values[idx])); - assertTrue(ord >= 0); - assertEquals(ss.ord(idx), ord); - } } - // Lookup random strings: - if (mode == Bytes.Mode.SORTED) { - final int numValues = ss.getValueCount(); - for (int i = 0; i < 1000; i++) { - BytesRef bytesValue = new BytesRef(_TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39))); - int ord = ss.getByValue(bytesValue); - if (ord >= 0) { - assertTrue(bytesValue - .bytesEquals(ss.getByOrd(ord, bytesRef))); - int count = 0; - for (int k = 0; k < 100; k++) { - if (bytesValue.utf8ToString().equals(values[2 * k])) { - assertEquals(ss.ord(2 * k), ord); - count++; - } - } - assertTrue(count > 0); - } else { - assert ord < 0; - int insertIndex = (-ord)-1; - if (insertIndex == 0) { - final BytesRef firstRef = ss.getByOrd(1, bytesRef); - // random string was before our first - assertTrue(firstRef.compareTo(bytesValue) > 0); - } else if (insertIndex == numValues) { - final BytesRef lastRef = ss.getByOrd(numValues-1, bytesRef); - // random string was after our last - assertTrue(lastRef.compareTo(bytesValue) < 0); - } else { - final BytesRef before = (BytesRef) ss.getByOrd(insertIndex-1, bytesRef) - .clone(); - BytesRef after = ss.getByOrd(insertIndex, bytesRef); - assertTrue(comp.compare(before, bytesValue) < 0); - assertTrue(comp.compare(bytesValue, after) < 0); - } - } - } - } } r.close(); @@ -194,14 +112,6 @@ public class TestDocValues extends LuceneTestCase { expectedTypes[i], source.type()); assertEquals(minMax[i][0], source.getInt(0)); assertEquals(minMax[i][1], source.getInt(1)); - ValuesEnum iEnum = getEnum(r); - assertEquals(i + " with min: " + minMax[i][0] + " max: " + minMax[i][1], - expectedTypes[i], iEnum.type()); - assertEquals(0, iEnum.nextDoc()); - assertEquals(minMax[i][0], iEnum.intsRef.get()); - assertEquals(1, iEnum.nextDoc()); - assertEquals(minMax[i][1], iEnum.intsRef.get()); - assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.nextDoc()); r.close(); dir.close(); @@ -373,37 +283,6 @@ public class TestDocValues extends LuceneTestCase { } } - for (int iter = 0; iter < 2; iter++) { - ValuesEnum iEnum = getEnum(r); - assertEquals(type, iEnum.type()); - LongsRef ints = iEnum.getInt(); - for (int i = 0; i < NUM_VALUES + additionalDocs; i++) { - assertEquals(i, iEnum.nextDoc()); - if (i < NUM_VALUES) { - assertEquals(values[i], ints.get()); - } else { - assertEquals(0, ints.get()); - } - } - assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.nextDoc()); - iEnum.close(); - } - - for (int iter = 0; iter < 2; iter++) { - ValuesEnum iEnum = getEnum(r); - assertEquals(type, iEnum.type()); - LongsRef ints = iEnum.getInt(); - for (int i = 0; i < NUM_VALUES + additionalDocs; i += 1 + random.nextInt(25)) { - assertEquals(i, iEnum.advance(i)); - if (i < NUM_VALUES) { - assertEquals(values[i], ints.get()); - } else { - assertEquals(0, ints.get()); - } - } - assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.advance(NUM_VALUES + additionalDocs)); - iEnum.close(); - } r.close(); dir.close(); } @@ -433,39 +312,9 @@ public class TestDocValues extends LuceneTestCase { for (int iter = 0; iter < 2; iter++) { Source s = getSource(r); for (int i = 0; i < NUM_VALUES; i++) { - assertEquals(values[i], s.getFloat(i), 0.0f); - } - } - - for (int iter = 0; iter < 2; iter++) { - ValuesEnum fEnum = getEnum(r); - FloatsRef floats = fEnum.getFloat(); - for (int i = 0; i < NUM_VALUES + additionalValues; i++) { - assertEquals(i, fEnum.nextDoc()); - if (i < NUM_VALUES) { - assertEquals(values[i], floats.get(), delta); - } else { - assertEquals(0.0d, floats.get(), delta); - } - } - assertEquals(ValuesEnum.NO_MORE_DOCS, fEnum.nextDoc()); - fEnum.close(); - } - for (int iter = 0; iter < 2; iter++) { - ValuesEnum fEnum = getEnum(r); - FloatsRef floats = fEnum.getFloat(); - for (int i = 0; i < NUM_VALUES + additionalValues; i += 1 + random.nextInt(25)) { - assertEquals(i, fEnum.advance(i)); - if (i < NUM_VALUES) { - assertEquals(values[i], floats.get(), delta); - } else { - assertEquals(0.0d, floats.get(), delta); - } + assertEquals("" + i, values[i], s.getFloat(i), 0.0f); } - assertEquals(ValuesEnum.NO_MORE_DOCS, fEnum.advance(NUM_VALUES + additionalValues)); - fEnum.close(); } - r.close(); dir.close(); } @@ -474,19 +323,19 @@ public class TestDocValues extends LuceneTestCase { runTestFloats(8, 0.0); } - private ValuesEnum getEnum(IndexDocValues values) throws IOException { - return random.nextBoolean() ? values.getEnum() : getSource(values).getEnum(); - } private Source getSource(IndexDocValues values) throws IOException { // getSource uses cache internally - return random.nextBoolean() ? values.load() : values.getSource(); - } - - private SortedSource getSortedSource(IndexDocValues values, - Comparator comparator) throws IOException { - // getSortedSource uses cache internally - return random.nextBoolean() ? values.loadSorted(comparator) : values - .getSortedSorted(comparator); + switch(random.nextInt(5)) { + case 3: + return values.load(); + case 2: + return values.getDirectSource(); + case 1: + return values.getSource(); + default: + return values.getSource(); + } } + } diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java index bff0c88..e6dde00 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java @@ -47,8 +47,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.FloatsRef; -import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; import org.junit.Before; @@ -136,7 +134,6 @@ public class TestDocValuesIndexing extends LuceneTestCase { Collections.shuffle(values, random); ValueType first = values.get(0); ValueType second = values.get(1); - String msg = "[first=" + first.name() + ", second=" + second.name() + "]"; // index first index Directory d_1 = newDirectory(); IndexWriter w_1 = new IndexWriter(d_1, writerConfig(random.nextBoolean())); @@ -171,36 +168,62 @@ public class TestDocValuesIndexing extends LuceneTestCase { // check values IndexReader merged = IndexReader.open(w, true); - ValuesEnum vE_1 = getValuesEnum(getDocValues(r_1, first.name())); - ValuesEnum vE_2 = getValuesEnum(getDocValues(r_2, second.name())); - ValuesEnum vE_1_merged = getValuesEnum(getDocValues(merged, first.name())); - ValuesEnum vE_2_merged = getValuesEnum(getDocValues(merged, second + Source source_1 = getSource(getDocValues(r_1, first.name())); + Source source_2 = getSource(getDocValues(r_2, second.name())); + Source source_1_merged = getSource(getDocValues(merged, first.name())); + Source source_2_merged = getSource(getDocValues(merged, second .name())); - switch (second) { // these variants don't advance over missing values - case BYTES_FIXED_STRAIGHT: - case BYTES_VAR_STRAIGHT: - case FLOAT_32: - case FLOAT_64: - case VAR_INTS: - case FIXED_INTS_16: - case FIXED_INTS_32: - case FIXED_INTS_64: - case FIXED_INTS_8: - assertEquals(msg, valuesPerIndex-1, vE_2_merged.advance(valuesPerIndex-1)); + for (int i = 0; i < r_1.maxDoc(); i++) { + switch (first) { + case BYTES_FIXED_DEREF: + case BYTES_FIXED_STRAIGHT: + case BYTES_VAR_DEREF: + case BYTES_VAR_STRAIGHT: + assertEquals(source_1.getBytes(i, new BytesRef()), + source_1_merged.getBytes(i, new BytesRef())); + break; + case FIXED_INTS_16: + case FIXED_INTS_32: + case FIXED_INTS_64: + case FIXED_INTS_8: + case VAR_INTS: + assertEquals(source_1.getInt(i), source_1_merged.getInt(i)); + break; + case FLOAT_32: + case FLOAT_64: + assertEquals(source_1.getFloat(i), source_1_merged.getFloat(i), 0.0d); + break; + default: + fail("unkonwn " + first); + } } - - for (int i = 0; i < valuesPerIndex; i++) { - assertEquals(msg, i, vE_1.nextDoc()); - assertEquals(msg, i, vE_1_merged.nextDoc()); - assertEquals(msg, i, vE_2.nextDoc()); - assertEquals(msg, i + valuesPerIndex, vE_2_merged.nextDoc()); + for (int i = r_1.maxDoc(); i < merged.maxDoc(); i++) { + switch (second) { + case BYTES_FIXED_DEREF: + case BYTES_FIXED_STRAIGHT: + case BYTES_VAR_DEREF: + case BYTES_VAR_STRAIGHT: + assertEquals(source_2.getBytes(i - r_1.maxDoc(), new BytesRef()), + source_2_merged.getBytes(i, new BytesRef())); + break; + case FIXED_INTS_16: + case FIXED_INTS_32: + case FIXED_INTS_64: + case FIXED_INTS_8: + case VAR_INTS: + assertEquals(source_2.getInt(i - r_1.maxDoc()), + source_2_merged.getInt(i)); + break; + case FLOAT_32: + case FLOAT_64: + assertEquals(source_2.getFloat(i - r_1.maxDoc()), + source_2_merged.getFloat(i), 0.0d); + break; + default: + fail("unkonwn " + first); + } } - assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_1.nextDoc()); - assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_2.nextDoc()); - assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_1_merged.advance(valuesPerIndex*2)); - assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_2_merged.nextDoc()); - // close resources r_1.close(); r_2.close(); @@ -260,22 +283,12 @@ public class TestDocValuesIndexing extends LuceneTestCase { assertEquals("index " + i, 0, value); } - ValuesEnum intsEnum = getValuesEnum(intsReader); - assertTrue(intsEnum.advance(base) >= base); - - intsEnum = getValuesEnum(intsReader); - LongsRef enumRef = intsEnum.getInt(); - int expected = 0; for (int i = base; i < r.numDocs(); i++, expected++) { while (deleted.get(expected)) { expected++; } - assertEquals("advance failed at index: " + i + " of " + r.numDocs() - + " docs", i, intsEnum.advance(i)); assertEquals(val + " mod: " + mod + " index: " + i, expected%mod, ints.getInt(i)); - assertEquals(expected%mod, enumRef.get()); - } } break; @@ -289,20 +302,11 @@ public class TestDocValuesIndexing extends LuceneTestCase { assertEquals(val + " failed for doc: " + i + " base: " + base, 0.0d, value, 0.0d); } - ValuesEnum floatEnum = getValuesEnum(floatReader); - assertTrue(floatEnum.advance(base) >= base); - - floatEnum = getValuesEnum(floatReader); - FloatsRef enumRef = floatEnum.getFloat(); int expected = 0; for (int i = base; i < r.numDocs(); i++, expected++) { while (deleted.get(expected)) { expected++; } - assertEquals("advance failed at index: " + i + " of " + r.numDocs() - + " docs base:" + base, i, floatEnum.advance(i)); - assertEquals(floatEnum.getClass() + " index " + i, 2.0 * expected, - enumRef.get(), 0.00001); assertEquals("index " + i, 2.0 * expected, floats.getFloat(i), 0.00001); } @@ -320,7 +324,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { w.close(); d.close(); } - + public void runTestIndexBytes(IndexWriterConfig cfg, boolean withDeletions) throws CorruptIndexException, LockObtainFailedException, IOException { final Directory d = newDirectory(); @@ -353,6 +357,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { switch (byteIndexValue) { case BYTES_VAR_STRAIGHT: case BYTES_FIXED_STRAIGHT: + case BYTES_FIXED_DEREF: // fixed straight returns bytesref with zero bytes all of fixed // length assertNotNull("expected none null - " + msg, br); @@ -365,23 +370,14 @@ public class TestDocValuesIndexing extends LuceneTestCase { } } break; - case BYTES_VAR_SORTED: - case BYTES_FIXED_SORTED: case BYTES_VAR_DEREF: - case BYTES_FIXED_DEREF: default: assertNotNull("expected none null - " + msg, br); - assertEquals(0, br.length); + assertEquals(byteIndexValue + "", 0, br.length); // make sure we advance at least until base - ValuesEnum bytesEnum = getValuesEnum(bytesReader); - final int advancedTo = bytesEnum.advance(0); - assertTrue(byteIndexValue.name() + " advanced failed base:" + base - + " advancedTo: " + advancedTo, base <= advancedTo); } } - ValuesEnum bytesEnum = getValuesEnum(bytesReader); - final BytesRef enumRef = bytesEnum.bytes(); // test the actual doc values added in this iteration assertEquals(base + numRemainingValues, r.numDocs()); int v = 0; @@ -393,17 +389,8 @@ public class TestDocValuesIndexing extends LuceneTestCase { upto += bytesSize; } BytesRef br = bytes.getBytes(i, new BytesRef()); - if (bytesEnum.docID() != i) { - assertEquals("seek failed for index " + i + " " + msg, i, bytesEnum - .advance(i)); - } assertTrue(msg, br.length > 0); for (int j = 0; j < br.length; j++, upto++) { - assertTrue(" enumRef not initialized " + msg, - enumRef.bytes.length > 0); - assertEquals( - "EnumRef Byte at index " + j + " doesn't match - " + msg, upto, - enumRef.bytes[enumRef.offset + j]); if (!(br.bytes.length > br.offset + j)) br = bytes.getBytes(i, new BytesRef()); assertTrue("BytesRef index exceeded [" + msg + "] offset: " @@ -446,33 +433,23 @@ public class TestDocValuesIndexing extends LuceneTestCase { } private Source getSource(IndexDocValues values) throws IOException { - Source source; - if (random.nextInt(10) == 0) { - source = values.load(); - } else { - // getSource uses cache internally - source = values.getSource(); + // getSource uses cache internally + switch(random.nextInt(5)) { + case 3: + return values.load(); + case 2: + return values.getDirectSource(); + case 1: + return values.getSource(); + default: + return values.getSource(); } - assertNotNull(source); - return source; } - private ValuesEnum getValuesEnum(IndexDocValues values) throws IOException { - ValuesEnum valuesEnum; - if (!(values instanceof MultiIndexDocValues) && random.nextInt(10) == 0) { - // TODO not supported by MultiDocValues yet! - valuesEnum = getSource(values).getEnum(); - } else { - valuesEnum = values.getEnum(); - - } - assertNotNull(valuesEnum); - return valuesEnum; - } private static EnumSet BYTES = EnumSet.of(ValueType.BYTES_FIXED_DEREF, - ValueType.BYTES_FIXED_SORTED, ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF, - ValueType.BYTES_VAR_SORTED, ValueType.BYTES_VAR_STRAIGHT); + ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF, + ValueType.BYTES_VAR_STRAIGHT); private static EnumSet NUMERICS = EnumSet.of(ValueType.VAR_INTS, ValueType.FIXED_INTS_16, ValueType.FIXED_INTS_32, diff --git a/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java index 107bf02..4933e73 100644 --- a/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java +++ b/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java @@ -115,12 +115,38 @@ public class TestPackedInts extends LuceneTestCase { assertEquals(fp, in.getFilePointer()); in.close(); } + + { // test reader iterator seek + IndexInput in = d.openInput("out.bin", newIOContext(random)); + PackedInts.SeekableReaderIterator intsEnum = PackedInts.getSeekableReaderIterator(in); + for (int i = 0; i < valueCount; i++) { + final String msg = "index=" + i + " ceil=" + ceil + " valueCount=" + + valueCount + " nbits=" + nbits + " for " + + intsEnum.getClass().getSimpleName(); + final int ord = random.nextInt(valueCount); + long seek = intsEnum.seek(ord); + assertEquals(msg, seek, values[ord]); + if (random.nextBoolean() && ord < valueCount-1) { + if (random.nextBoolean()) { + assertEquals(msg, values[ord+1], intsEnum.advance(ord+1)); + } else { + assertEquals(msg, values[ord+1], intsEnum.next()); + } + } + } + if (intsEnum.ord() < valueCount - 1) + assertEquals(values[valueCount - 1], intsEnum + .advance(valueCount - 1)); + assertEquals(valueCount - 1, intsEnum.ord()); + assertEquals(fp, in.getFilePointer()); + in.close(); + } ceil *= 2; d.close(); } } } - + public void testControlledEquality() { final int VALUE_COUNT = 255; final int BITS_PER_VALUE = 8;