diff --git a/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java b/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java index e45248c..a696505 100644 --- a/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java +++ b/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java @@ -317,8 +317,10 @@ public class IndexDocValuesField extends Field implements PerDocFieldValues { final String value; switch (type) { case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: value = "bytes:bytes.utf8ToString();"; break; @@ -351,8 +353,10 @@ public class IndexDocValuesField extends Field implements PerDocFieldValues { final IndexDocValuesField valField = new IndexDocValuesField(field.name(), field.fieldType(), field.stringValue()); switch (type) { case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: BytesRef ref = field.isBinary() ? field.binaryValue() : new BytesRef(field.stringValue()); valField.setBytes(ref, type); diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 0620224..2982203 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -1074,8 +1074,10 @@ public class CheckIndex { while (values.nextDoc() != ValuesEnum.NO_MORE_DOCS) { switch (fieldInfo.docValues) { case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: values.bytes(); break; diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index c8a0646..6108aeb 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -652,24 +652,30 @@ public final class FieldInfos implements Iterable { case BYTES_FIXED_DEREF: b = 5; break; - case BYTES_VAR_STRAIGHT: + case BYTES_FIXED_SORTED: b = 6; break; - case BYTES_VAR_DEREF: + case BYTES_VAR_STRAIGHT: b = 7; break; - case FIXED_INTS_16: + case BYTES_VAR_DEREF: b = 8; break; - case FIXED_INTS_32: + case BYTES_VAR_SORTED: b = 9; break; - case FIXED_INTS_64: + case FIXED_INTS_16: b = 10; break; - case FIXED_INTS_8: + case FIXED_INTS_32: b = 11; break; + case FIXED_INTS_64: + b = 12; + break; + case FIXED_INTS_8: + b = 13; + break; default: throw new IllegalStateException("unhandled indexValues type " + fi.docValues); @@ -748,21 +754,27 @@ public final class FieldInfos implements Iterable { docValuesType = ValueType.BYTES_FIXED_DEREF; break; case 6: - docValuesType = ValueType.BYTES_VAR_STRAIGHT; + docValuesType = ValueType.BYTES_FIXED_SORTED; break; case 7: - docValuesType = ValueType.BYTES_VAR_DEREF; + docValuesType = ValueType.BYTES_VAR_STRAIGHT; break; case 8: - docValuesType = ValueType.FIXED_INTS_16; + docValuesType = ValueType.BYTES_VAR_DEREF; break; case 9: - docValuesType = ValueType.FIXED_INTS_32; + docValuesType = ValueType.BYTES_VAR_SORTED; break; case 10: - docValuesType = ValueType.FIXED_INTS_64; + docValuesType = ValueType.FIXED_INTS_16; break; case 11: + docValuesType = ValueType.FIXED_INTS_32; + break; + case 12: + docValuesType = ValueType.FIXED_INTS_64; + break; + case 13: docValuesType = ValueType.FIXED_INTS_8; break; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java index dbf6ec2..0e1e21d 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs; import java.io.Closeable; import java.io.IOException; import java.util.Collection; +import java.util.Comparator; import java.util.Map; import java.util.TreeMap; @@ -32,6 +33,7 @@ import org.apache.lucene.index.values.Ints; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; /** * Abstract base class for PerDocValues implementations @@ -57,6 +59,10 @@ public abstract class DocValuesReaderBase extends PerDocValues { return docValues().keySet(); } + public Comparator getComparator() throws IOException { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, String segment, int docCount, Directory dir, int codecId, IOContext context) @@ -119,13 +125,17 @@ public abstract class DocValuesReaderBase extends PerDocValues { case FLOAT_64: return Floats.getValues(dir, id, docCount, context); case BYTES_FIXED_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, context); + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context); case BYTES_FIXED_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, context); + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context); + case BYTES_FIXED_SORTED: + return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context); case BYTES_VAR_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, context); + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context); case BYTES_VAR_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, context); + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context); + case BYTES_VAR_SORTED: + return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context); default: throw new IllegalStateException("unrecognized index values mode " + type); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java index bc37527..21cf563 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java @@ -18,12 +18,14 @@ package org.apache.lucene.index.codecs; */ import java.io.IOException; +import java.util.Comparator; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.values.Writer; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; /** @@ -53,10 +55,14 @@ public abstract class DocValuesWriterBase extends PerDocConsumer { public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), docValuesId(segmentName, codecId, field.number), - getDirectory(), bytesUsed, context); + getDirectory(), getComparator(), bytesUsed, context); } public static String docValuesId(String segmentsName, int codecID, int fieldId) { return segmentsName + "_" + codecID + "-" + fieldId; } + + public Comparator getComparator() throws IOException { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java index 2bdae4b..1419950 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java @@ -56,6 +56,8 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_FIXED_SORTED: case BYTES_VAR_STRAIGHT: files.add(IndexFileNames.segmentFileName(filename, "", Writer.INDEX_EXTENSION)); diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index 64844fd..70f32fa 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -20,9 +20,11 @@ package org.apache.lucene.index.values; /** Base class for specific Bytes Reader/Writer implementations */ import java.io.IOException; import java.util.Collection; +import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.index.values.IndexDocValues.SourceEnum; import org.apache.lucene.store.DataOutput; @@ -82,6 +84,10 @@ public final class Bytes { * Mode for dereferenced stored bytes */ DEREF, + /** + * Mode for sorted stored bytes + */ + SORTED }; /** @@ -110,22 +116,30 @@ public final class Bytes { * @throws IOException * if the files for the writer can not be created. */ - public static Writer getWriter(Directory dir, String id, Mode mode, boolean fixedSize, Counter bytesUsed, IOContext context) + public static Writer getWriter(Directory dir, String id, Mode mode, + Comparator comp, boolean fixedSize, Counter bytesUsed, IOContext context) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? + if (comp == null) { + comp = BytesRef.getUTF8SortedAsUnicodeComparator(); + } if (fixedSize) { if (mode == Mode.STRAIGHT) { return new FixedStraightBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); + } else if (mode == Mode.SORTED) { + return new FixedSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context); } } else { if (mode == Mode.STRAIGHT) { return new VarStraightBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); + } else if (mode == Mode.SORTED) { + return new VarSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context); } } @@ -155,7 +169,7 @@ public final class Bytes { * if an {@link IOException} occurs */ public static IndexDocValues getValues(Directory dir, String id, Mode mode, - boolean fixedSize, int maxDoc, IOContext context) throws IOException { + boolean fixedSize, int maxDoc, Comparator sortComparator, IOContext context) throws IOException { // TODO -- I can peek @ header to determing fixed/mode? if (fixedSize) { @@ -163,12 +177,16 @@ public final class Bytes { return new FixedStraightBytesImpl.Reader(dir, id, maxDoc, context); } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Reader(dir, id, maxDoc, context); + } else if (mode == Mode.SORTED) { + return new FixedSortedBytesImpl.Reader(dir, id, maxDoc, context); } } else { if (mode == Mode.STRAIGHT) { return new VarStraightBytesImpl.Reader(dir, id, maxDoc, context); } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Reader(dir, id, maxDoc, context); + } else if (mode == Mode.SORTED) { + return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator, context); } } @@ -270,6 +288,111 @@ public final class Bytes { } + static abstract class BytesSortedSourceBase extends SortedSource { + private final PagedBytes pagedBytes; + private final Comparator comp; + protected final PackedInts.Reader docToOrdIndex; + private final ValueType type; + + protected final IndexInput datIn; + protected final IndexInput idxIn; + protected final BytesRef defaultValue = new BytesRef(); + protected final static int PAGED_BYTES_BITS = 15; + protected final PagedBytes.Reader data; + + + protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn, + Comparator comp, long bytesToRead, ValueType type) throws IOException { + this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type); + } + + protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn, + Comparator comp, PagedBytes pagedBytes, long bytesToRead,ValueType type) + throws IOException { + assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: " + + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer(); + this.datIn = datIn; + this.pagedBytes = pagedBytes; + this.pagedBytes.copy(datIn, bytesToRead); + data = pagedBytes.freeze(true); + this.idxIn = idxIn; + this.comp = comp == null ? BytesRef.getUTF8SortedAsUnicodeComparator() + : comp; + docToOrdIndex = PackedInts.getReader(idxIn); + this.type = type; + + } + + @Override + public int ord(int docID) { + return (int) docToOrdIndex.get(docID) -1; + } + + @Override + public BytesRef getByOrd(int ord, BytesRef bytesRef) { + assert ord >= 0; + return deref(ord, bytesRef); + } + + protected void closeIndexInput() throws IOException { + IOUtils.close(datIn, idxIn); + } + + /** + * Returns the largest doc id + 1 in this doc values source + */ + public int maxDoc() { + return docToOrdIndex.size(); + } + /** + * Copies the value for the given ord to the given {@link BytesRef} and + * returns it. + */ + protected abstract BytesRef deref(int ord, BytesRef bytesRef); + + protected int binarySearch(BytesRef b, BytesRef bytesRef, int low, + int high) { + int mid = 0; + while (low <= high) { + mid = (low + high) >>> 1; + deref(mid, bytesRef); + final int cmp = comp.compare(bytesRef, b); + if (cmp < 0) { + low = mid + 1; + } else if (cmp > 0) { + high = mid - 1; + } else { + return mid; + } + } + assert comp.compare(bytesRef, b) != 0; + return -(low + 1); + } + + @Override + public ValuesEnum getEnum(AttributeSource attrSource) throws IOException { + return new SourceEnum(attrSource, type(), this, maxDoc()) { + + @Override + public int advance(int target) throws IOException { + if (target >= numDocs) { + return pos = NO_MORE_DOCS; + } + while (source.getBytes(target, bytesRef).length == 0) { + if (++target >= numDocs) { + return pos = NO_MORE_DOCS; + } + } + return pos = target; + } + }; + } + + @Override + public ValueType type() { + return type; + } + } // TODO: open up this API?! static abstract class BytesWriterBase extends Writer { diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java new file mode 100644 index 0000000..b5969ae --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java @@ -0,0 +1,143 @@ +package org.apache.lucene.index.values; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase; +import org.apache.lucene.index.values.Bytes.BytesReaderBase; +import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; +import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; + +// Stores fixed-length byte[] by deref, ie when two docs +// have the same value, they store only 1 byte[] + +/** + * @lucene.experimental + */ +class FixedSortedBytesImpl { + + static final String CODEC_NAME = "FixedSortedBytes"; + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + static class Writer extends DerefBytesWriterBase { + private final Comparator comp; + + public Writer(Directory dir, String id, Comparator comp, + Counter bytesUsed, IOContext context) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); + this.comp = comp; + } + + // Important that we get docCount, in case there were + // some last docs that we didn't see + @Override + public void finishInternal(int docCount) throws IOException { + final IndexOutput datOut = getOrCreateDataOut(); + final int count = hash.size(); + final int[] address = new int[count+1]; // addr 0 is default values + datOut.writeInt(size); + if (size != -1) { + final int[] sortedEntries = hash.sort(comp); + // first dump bytes data, recording address as we go + final BytesRef bytesRef = new BytesRef(size); + for (int i = 0; i < count; i++) { + final int e = sortedEntries[i]; + final BytesRef bytes = hash.get(e, bytesRef); + assert bytes.length == size; + datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length); + address[e + 1] = 1 + i; + } + } + final IndexOutput idxOut = getOrCreateIndexOut(); + idxOut.writeInt(count); + writeIndex(idxOut, docCount, count, address, docToEntry); + } + } + + public static class Reader extends BytesReaderBase { + private final int size; + private final int numValuesStored; + + public Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException { + super(dir, id, CODEC_NAME, VERSION_START, true, context); + size = datIn.readInt(); + numValuesStored = idxIn.readInt(); + } + + @Override + public org.apache.lucene.index.values.IndexDocValues.Source load() + throws IOException { + return loadSorted(null); + } + + @Override + public SortedSource loadSorted(Comparator comp) + throws IOException { + return new Source(cloneData(), cloneIndex(), size, numValuesStored, comp); + } + + private static class Source extends BytesSortedSourceBase { + private final int valueCount; + private final int size; + + public Source(IndexInput datIn, IndexInput idxIn, int size, + int numValues, Comparator comp) throws IOException { + super(datIn, idxIn, comp, size * numValues, ValueType.BYTES_FIXED_SORTED); + this.size = size; + this.valueCount = numValues; + closeIndexInput(); + } + + @Override + public int getByValue(BytesRef bytes, BytesRef tmpRef) { + return binarySearch(bytes, tmpRef, 0, valueCount - 1); + } + + @Override + public int getValueCount() { + return valueCount; + } + + @Override + protected BytesRef deref(int ord, BytesRef bytesRef) { + return data.fillSlice(bytesRef, (ord * size), size); + } + } + + @Override + public ValuesEnum getEnum(AttributeSource source) throws IOException { + // do unsorted + return new DerefBytesEnum(source, cloneData(), cloneIndex(), size); + } + + @Override + public ValueType type() { + return ValueType.BYTES_FIXED_SORTED; + } + } +} diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java index 96a7393..305a076 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java @@ -120,6 +120,54 @@ public abstract class IndexDocValues implements Closeable { } /** + * Returns a {@link SortedSource} instance for this {@link IndexDocValues} field + * instance like {@link #getSource()}. + *

+ * This method will return null iff this {@link IndexDocValues} represent a + * {@link Source} instead of a {@link SortedSource}. + */ + public SortedSource getSortedSorted(Comparator comparator) + throws IOException { + return cache.loadSorted(this, comparator); + } + + /** + * Returns a {@link SortedSource} instance using a default {@link BytesRef} + * comparator for this {@link IndexDocValues} field instance like + * {@link #getSource()}. + *

+ * This method will return null iff this {@link IndexDocValues} represent a + * {@link Source} instead of a {@link SortedSource}. + */ + public SortedSource getSortedSorted() throws IOException { + return getSortedSorted(null); + } + + /** + * Loads and returns a {@link SortedSource} instance for this + * {@link IndexDocValues} field instance like {@link #load()}. + *

+ * This method will return null iff this {@link IndexDocValues} represent a + * {@link Source} instead of a {@link SortedSource}. + */ + public SortedSource loadSorted(Comparator comparator) + throws IOException { + throw new UnsupportedOperationException(); + } + + /** + * Loads and returns a {@link SortedSource} instance using a default + * {@link BytesRef} comparator for this {@link IndexDocValues} field instance + * like {@link #load()}. + *

+ * This method will return null iff this {@link IndexDocValues} represent a + * {@link Source} instead of a {@link SortedSource}. + */ + public SortedSource loadSorted() throws IOException { + return loadSorted(null); + } + + /** * Returns the {@link ValueType} of this {@link IndexDocValues} instance */ public abstract ValueType type(); @@ -295,4 +343,65 @@ public abstract class IndexDocValues implements Closeable { return advance(pos + 1); } } + + /** + * A sorted variant of {@link Source} for byte[] values per document. + *

+ * Note: {@link ValuesEnum} obtained from a {@link SortedSource} will + * enumerate values in document order and not in sorted order. + */ + public static abstract class SortedSource extends Source { + + @Override + public BytesRef getBytes(int docID, BytesRef bytesRef) { + final int ord = ord(docID); + if (ord < 0) { + bytesRef.length = 0; + } else { + getByOrd(ord , bytesRef); + } + return bytesRef; + } + + /** + * Returns ord for specified docID. If this docID had not been added to the + * Writer, the ord is 0. Ord is dense, ie, starts at 0, then increments by 1 + * for the next (as defined by {@link Comparator} value. + */ + public abstract int ord(int docID); + + /** Returns value for specified ord. */ + public abstract BytesRef getByOrd(int ord, BytesRef bytesRef); + + + /** + * Finds the ordinal whose value is greater or equal to the given value. + * + * @return the given values ordinal if found or otherwise + * (-(ord)-1), defined as the ordinal of the first + * element that is greater than the given value. This guarantees + * that the return value will always be >= 0 if the given value + * is found. + * + */ + public final int getByValue(BytesRef value) { + return getByValue(value, new BytesRef()); + } + + /** + * Performs a lookup by value. + * + * @param value + * the value to look up + * @param tmpRef + * a temporary {@link BytesRef} instance used to compare internal + * values to the given value. Must not be null + * @return the given values ordinal if found or otherwise + * (-(ord)-1), defined as the ordinal of the first + * element that is greater than the given value. This guarantees + * that the return value will always be >= 0 if the given value + * is found. + */ + public abstract int getByValue(BytesRef value, BytesRef tmpRef); + } } diff --git a/lucene/src/java/org/apache/lucene/index/values/SourceCache.java b/lucene/src/java/org/apache/lucene/index/values/SourceCache.java index afa0c80..7080006 100644 --- a/lucene/src/java/org/apache/lucene/index/values/SourceCache.java +++ b/lucene/src/java/org/apache/lucene/index/values/SourceCache.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Comparator; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -62,6 +63,16 @@ public abstract class SourceCache { public abstract Source load(IndexDocValues values) throws IOException; /** + * Atomically loads a {@link SortedSource} into the cache from the given + * {@link IndexDocValues} and returns it iff no other {@link SortedSource} has + * already been cached. Otherwise the cached source is returned. + *

+ * This method will not return null + */ + public abstract SortedSource loadSorted(IndexDocValues values, + Comparator comp) throws IOException; + + /** * Atomically invalidates the cached {@link Source} and {@link SortedSource} * instances if any and empties the cache. */ @@ -83,6 +94,7 @@ public abstract class SourceCache { */ public static final class DirectSourceCache extends SourceCache { private Source ref; + private SortedSource sortedRef; public synchronized Source load(IndexDocValues values) throws IOException { if (ref == null) { @@ -91,8 +103,17 @@ public abstract class SourceCache { return ref; } + public synchronized SortedSource loadSorted(IndexDocValues values, + Comparator comp) throws IOException { + if (sortedRef == null) { + sortedRef = values.loadSorted(comp); + } + return sortedRef; + } + public synchronized void invalidate(IndexDocValues values) { ref = null; + sortedRef = null; } } diff --git a/lucene/src/java/org/apache/lucene/index/values/ValueType.java b/lucene/src/java/org/apache/lucene/index/values/ValueType.java index f843453..974d496 100644 --- a/lucene/src/java/org/apache/lucene/index/values/ValueType.java +++ b/lucene/src/java/org/apache/lucene/index/values/ValueType.java @@ -18,6 +18,7 @@ package org.apache.lucene.index.values; */ import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.packed.PackedInts; @@ -166,6 +167,26 @@ public enum ValueType { BYTES_FIXED_DEREF, /** + * A fixed length pre-sorted byte[] variant. Fields with this type only + * store distinct byte values and store an additional offset pointer per + * document to dereference the shared byte[]. The stored + * byte[] is presorted, by default by unsigned byte order, + * and allows access via document id, ordinal and by-value. + * Use this type if your documents may share the same byte[]. + *

+ * NOTE: Fields of this type will not store values for documents without and + * explicitly provided value. If a documents value is accessed while no + * explicit value is stored the returned {@link BytesRef} will be a 0-length + * reference. In turn, {@link ValuesEnum} instances will skip over documents + * without an explicit value assigned. Custom default values must be assigned + * explicitly. + *

+ * + * @see SortedSource + */ + BYTES_FIXED_SORTED, + + /** * Variable length straight stored byte[] variant. All bytes are * stored sequentially for compactness. Usage of this type via the * disk-resident API might yield performance degradation since no additional @@ -195,4 +216,21 @@ public enum ValueType { *

*/ BYTES_VAR_DEREF, + + /** + * A variable length pre-sorted byte[] variant. Just like + * {@link #BYTES_FIXED_SORTED}, but allowing each + * document's value to be a different length. + *

+ * NOTE: Fields of this type will not store values for documents without and + * explicitly provided value. If a documents value is accessed while no + * explicit value is stored the returned {@link BytesRef} will be a 0-length + * reference. In turn, {@link ValuesEnum} instances will skip over documents + * without an explicit value assigned. Custom default values must be assigned + * explicitly. + *

+ * + * @see SortedSource + */ + BYTES_VAR_SORTED } diff --git a/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java new file mode 100644 index 0000000..36caf0b --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java @@ -0,0 +1,247 @@ +package org.apache.lucene.index.values; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase; +import org.apache.lucene.index.values.Bytes.BytesReaderBase; +import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; + +// Stores variable-length byte[] by deref, ie when two docs +// have the same value, they store only 1 byte[] and both +// docs reference that single source + +/** + * @lucene.experimental + */ +class VarSortedBytesImpl { + + static final String CODEC_NAME = "VarDerefBytes"; + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + final static class Writer extends DerefBytesWriterBase { + private final Comparator comp; + + public Writer(Directory dir, String id, Comparator comp, + Counter bytesUsed, IOContext context) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); + this.comp = comp; + } + + @Override + protected void checkSize(BytesRef bytes) { + // allow var bytes sizes + } + + // Important that we get docCount, in case there were + // some last docs that we didn't see + @Override + public void finishInternal(int docCount) throws IOException { + final int count = hash.size(); + final IndexOutput datOut = getOrCreateDataOut(); + long offset = 0; + long lastOffset = 0; + final int[] index = new int[count+1]; + final long[] offsets = new long[count]; + final int[] sortedEntries = hash.sort(comp); + // first dump bytes data, recording index & offset as + // we go + for (int i = 0; i < count; i++) { + final int e = sortedEntries[i]; + offsets[i] = offset; + index[e+1] = 1 + i; + + final BytesRef bytes = hash.get(e, new BytesRef()); + // TODO: we could prefix code... + datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length); + lastOffset = offset; + offset += bytes.length; + } + + final IndexOutput idxOut = getOrCreateIndexOut(); + // total bytes of data + idxOut.writeLong(offset); + // write index -- first doc -> 1+ord + writeIndex(idxOut, docCount, count, index, docToEntry); + // next ord (0-based) -> offset + PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count, + PackedInts.bitsRequired(lastOffset)); + for (int i = 0; i < count; i++) { + offsetWriter.add(offsets[i]); + } + offsetWriter.finish(); + } + } + + public static class Reader extends BytesReaderBase { + + private final Comparator defaultComp; + + Reader(Directory dir, String id, int maxDoc, + Comparator comparator, IOContext context) throws IOException { + super(dir, id, CODEC_NAME, VERSION_START, true, context); + this.defaultComp = comparator; + } + + @Override + public org.apache.lucene.index.values.IndexDocValues.Source load() + throws IOException { + return loadSorted(defaultComp); + } + + @Override + public SortedSource loadSorted(Comparator comp) + throws IOException { + IndexInput indexIn = cloneIndex(); + return new Source(cloneData(), indexIn, comp, indexIn.readLong()); + } + + private static class Source extends BytesSortedSourceBase { + private final PackedInts.Reader ordToOffsetIndex; // 0-based + private final long totBytes; + private final int valueCount; + + public Source(IndexInput datIn, IndexInput idxIn, + Comparator comp, long dataLength) throws IOException { + super(datIn, idxIn, comp, dataLength, ValueType.BYTES_VAR_SORTED); + totBytes = dataLength; + ordToOffsetIndex = PackedInts.getReader(idxIn); + valueCount = ordToOffsetIndex.size(); + closeIndexInput(); + } + + @Override + public int getByValue(BytesRef bytes, BytesRef tmpRef) { + return binarySearch(bytes, tmpRef, 0, valueCount - 1); + } + + @Override + public int getValueCount() { + return valueCount; + } + + // ord is 0-based + @Override + protected BytesRef deref(int ord, BytesRef bytesRef) { + final long nextOffset; + if (ord == valueCount - 1) { + nextOffset = totBytes; + } else { + nextOffset = ordToOffsetIndex.get(1 + ord); + } + final long offset = ordToOffsetIndex.get(ord); + data.fillSlice(bytesRef, offset, (int) (nextOffset - offset)); + return bytesRef; + } + } + + @Override + public ValuesEnum getEnum(AttributeSource source) throws IOException { + return new VarSortedBytesEnum(source, cloneData(), cloneIndex()); + } + + private static class VarSortedBytesEnum extends ValuesEnum { + private PackedInts.Reader docToOrdIndex; + private PackedInts.Reader ordToOffsetIndex; + private IndexInput idxIn; + private IndexInput datIn; + private int valueCount; + private long totBytes; + private int docCount; + private int pos = -1; + private final long fp; + + protected VarSortedBytesEnum(AttributeSource source, IndexInput datIn, + IndexInput idxIn) throws IOException { + super(source, ValueType.BYTES_VAR_SORTED); + totBytes = idxIn.readLong(); + // keep that in memory to prevent lots of disk seeks + docToOrdIndex = PackedInts.getReader(idxIn); + ordToOffsetIndex = PackedInts.getReader(idxIn); + valueCount = ordToOffsetIndex.size(); + docCount = docToOrdIndex.size(); + fp = datIn.getFilePointer(); + this.idxIn = idxIn; + this.datIn = datIn; + } + + @Override + public void close() throws IOException { + idxIn.close(); + datIn.close(); + } + + @Override + public int advance(int target) throws IOException { + if (target >= docCount) { + return pos = NO_MORE_DOCS; + } + int ord; + while ((ord = (int) docToOrdIndex.get(target)) == 0) { + if (++target >= docCount) { + return pos = NO_MORE_DOCS; + } + } + final long offset = ordToOffsetIndex.get(--ord); + final long nextOffset; + if (ord == valueCount - 1) { + nextOffset = totBytes; + } else { + nextOffset = ordToOffsetIndex.get(1 + ord); + } + final int length = (int) (nextOffset - offset); + datIn.seek(fp + offset); + if (bytesRef.bytes.length < length) + bytesRef.grow(length); + datIn.readBytes(bytesRef.bytes, 0, length); + bytesRef.length = length; + bytesRef.offset = 0; + return pos = target; + } + + @Override + public int docID() { + return pos; + } + + @Override + public int nextDoc() throws IOException { + if (pos >= docCount) { + return pos = NO_MORE_DOCS; + } + return advance(pos + 1); + } + } + + @Override + public ValueType type() { + return ValueType.BYTES_VAR_SORTED; + } + } +} diff --git a/lucene/src/java/org/apache/lucene/index/values/Writer.java b/lucene/src/java/org/apache/lucene/index/values/Writer.java index a341d16..db22b12 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Writer.java +++ b/lucene/src/java/org/apache/lucene/index/values/Writer.java @@ -17,6 +17,7 @@ package org.apache.lucene.index.values; * limitations under the License. */ import java.io.IOException; +import java.util.Comparator; import org.apache.lucene.index.codecs.DocValuesConsumer; import org.apache.lucene.store.Directory; @@ -192,7 +193,10 @@ public abstract class Writer extends DocValuesConsumer { * @throws IOException */ public static Writer create(ValueType type, String id, Directory directory, - Counter bytesUsed, IOContext context) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context) throws IOException { + if (comp == null) { + comp = BytesRef.getUTF8SortedAsUnicodeComparator(); + } switch (type) { case FIXED_INTS_16: case FIXED_INTS_32: @@ -205,17 +209,24 @@ public abstract class Writer extends DocValuesConsumer { case FLOAT_64: return Floats.getWriter(directory, id, 8, bytesUsed, context); case BYTES_FIXED_STRAIGHT: - return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, + return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, true, bytesUsed, context); case BYTES_FIXED_DEREF: - return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, + return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, true, + bytesUsed, context); + case BYTES_FIXED_SORTED: + return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, true, bytesUsed, context); case BYTES_VAR_STRAIGHT: - return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, + return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, false, bytesUsed, context); case BYTES_VAR_DEREF: - return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, + return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, false, bytesUsed, context); + case BYTES_VAR_SORTED: + return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false, + bytesUsed, context); + default: throw new IllegalArgumentException("Unknown Values: " + type); } diff --git a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java index 7d61ab6..b02ee56 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java @@ -176,6 +176,7 @@ public class RandomIndexWriter implements Closeable { IndexDocValuesField docValuesField = new IndexDocValuesField(name); switch (type) { case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: case BYTES_FIXED_STRAIGHT: final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength); BytesRef fixedRef = new BytesRef(randomUnicodeString); @@ -188,6 +189,7 @@ public class RandomIndexWriter implements Closeable { docValuesField.setBytes(fixedRef, type); break; case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: case BYTES_VAR_STRAIGHT: BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200)); docValuesField.setBytes(ref, type); diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java index 479b35d..35c1586 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java @@ -18,7 +18,9 @@ package org.apache.lucene.index.values; */ import java.io.IOException; +import java.util.Comparator; +import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -44,14 +46,22 @@ public class TestDocValues extends LuceneTestCase { runTestBytes(Bytes.Mode.DEREF, false); } + public void testBytesSorted() throws IOException { + runTestBytes(Bytes.Mode.SORTED, true); + runTestBytes(Bytes.Mode.SORTED, false); + } + public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize) throws IOException { final BytesRef bytesRef = new BytesRef(); + final Comparator comp = mode == Bytes.Mode.SORTED ? BytesRef + .getUTF8SortedAsUnicodeComparator() : null; + Directory dir = newDirectory(); final Counter trackBytes = Counter.newCounter(); - Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, trackBytes, newIOContext(random)); + Writer w = Bytes.getWriter(dir, "test", mode, comp, fixedSize, trackBytes, newIOContext(random)); int maxDoc = 220; final String[] values = new String[maxDoc]; final int fixedLength = 1 + atLeast(50); @@ -71,7 +81,7 @@ public class TestDocValues extends LuceneTestCase { w.finish(maxDoc); assertEquals(0, trackBytes.get()); - IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, newIOContext(random)); + IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, comp, newIOContext(random)); for (int iter = 0; iter < 2; iter++) { ValuesEnum bytesEnum = getEnum(r); assertNotNull("enum is null", bytesEnum); @@ -93,15 +103,68 @@ public class TestDocValues extends LuceneTestCase { // Verify we can load source twice: for (int iter = 0; iter < 2; iter++) { Source s; + IndexDocValues.SortedSource ss; + if (mode == Bytes.Mode.SORTED) { + // default is unicode so we can simply pass null here + s = ss = getSortedSource(r, random.nextBoolean() ? comp : null); + } else { s = getSource(r); + ss = null; + } for (int i = 0; i < 100; i++) { final int idx = 2 * i; assertNotNull("doc " + idx + "; value=" + values[idx], s.getBytes(idx, bytesRef)); assertEquals("doc " + idx, values[idx], s.getBytes(idx, bytesRef) .utf8ToString()); + if (ss != null) { + assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx), + bytesRef).utf8ToString()); + int ord = ss + .getByValue(new BytesRef(values[idx])); + assertTrue(ord >= 0); + assertEquals(ss.ord(idx), ord); + } } + // Lookup random strings: + if (mode == Bytes.Mode.SORTED) { + final int numValues = ss.getValueCount(); + for (int i = 0; i < 1000; i++) { + BytesRef bytesValue = new BytesRef(_TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39))); + int ord = ss.getByValue(bytesValue); + if (ord >= 0) { + assertTrue(bytesValue + .bytesEquals(ss.getByOrd(ord, bytesRef))); + int count = 0; + for (int k = 0; k < 100; k++) { + if (bytesValue.utf8ToString().equals(values[2 * k])) { + assertEquals(ss.ord(2 * k), ord); + count++; + } + } + assertTrue(count > 0); + } else { + assert ord < 0; + int insertIndex = (-ord)-1; + if (insertIndex == 0) { + final BytesRef firstRef = ss.getByOrd(1, bytesRef); + // random string was before our first + assertTrue(firstRef.compareTo(bytesValue) > 0); + } else if (insertIndex == numValues) { + final BytesRef lastRef = ss.getByOrd(numValues-1, bytesRef); + // random string was after our last + assertTrue(lastRef.compareTo(bytesValue) < 0); + } else { + final BytesRef before = (BytesRef) ss.getByOrd(insertIndex-1, bytesRef) + .clone(); + BytesRef after = ss.getByOrd(insertIndex, bytesRef); + assertTrue(comp.compare(before, bytesValue) < 0); + assertTrue(comp.compare(bytesValue, after) < 0); + } + } + } + } } r.close(); @@ -419,4 +482,11 @@ public class TestDocValues extends LuceneTestCase { // getSource uses cache internally return random.nextBoolean() ? values.load() : values.getSource(); } + + private SortedSource getSortedSource(IndexDocValues values, + Comparator comparator) throws IOException { + // getSortedSource uses cache internally + return random.nextBoolean() ? values.loadSorted(comparator) : values + .getSortedSorted(comparator); + } } diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java index 646529b..bff0c88 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java @@ -365,6 +365,8 @@ public class TestDocValuesIndexing extends LuceneTestCase { } } break; + case BYTES_VAR_SORTED: + case BYTES_FIXED_SORTED: case BYTES_VAR_DEREF: case BYTES_FIXED_DEREF: default: @@ -469,8 +471,8 @@ public class TestDocValuesIndexing extends LuceneTestCase { } private static EnumSet BYTES = EnumSet.of(ValueType.BYTES_FIXED_DEREF, - ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF, - ValueType.BYTES_VAR_STRAIGHT); + ValueType.BYTES_FIXED_SORTED, ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF, + ValueType.BYTES_VAR_SORTED, ValueType.BYTES_VAR_STRAIGHT); private static EnumSet NUMERICS = EnumSet.of(ValueType.VAR_INTS, ValueType.FIXED_INTS_16, ValueType.FIXED_INTS_32,