Index: lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.util.Bits;
@@ -40,6 +41,7 @@
protected final BytesRef spare = new BytesRef();
+ protected abstract Type getType();
/**
* Adds the given {@link IndexableField} instance to this
* {@link DocValuesConsumer}
@@ -110,7 +112,7 @@
final Source source = reader.getDirectSource();
assert source != null;
int docID = docBase;
- final DocValues.Type type = reader.type();
+ final Type type = getType();
final Field scratchField;
switch(type) {
case VAR_INTS:
@@ -160,7 +162,7 @@
*/
protected void mergeDoc(Field scratchField, Source source, int docID, int sourceDoc)
throws IOException {
- switch(source.type()) {
+ switch(getType()) {
case BYTES_FIXED_DEREF:
case BYTES_FIXED_SORTED:
case BYTES_FIXED_STRAIGHT:
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (working copy)
@@ -242,8 +242,8 @@
private final IOContext context;
protected BytesWriterBase(Directory dir, String id, String codecName,
- int version, Counter bytesUsed, IOContext context) throws IOException {
- super(bytesUsed);
+ int version, Counter bytesUsed, IOContext context, Type type) throws IOException {
+ super(bytesUsed, type);
this.id = id;
this.dir = dir;
this.codecName = codecName;
@@ -292,25 +292,11 @@
}
return idxOut;
}
- /**
- * Must be called only with increasing docIDs. It's OK for some docIDs to be
- * skipped; they will be filled with 0 bytes.
- */
- protected
- abstract void add(int docID, BytesRef bytes) throws IOException;
+
@Override
public abstract void finish(int docCount) throws IOException;
- @Override
- protected void mergeDoc(Field scratchField, Source source, int docID, int sourceDoc) throws IOException {
- add(docID, source.getBytes(sourceDoc, bytesRef));
- }
-
- @Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.binaryValue());
- }
}
/**
@@ -393,22 +379,22 @@
protected long maxBytes = 0;
protected DerefBytesWriterBase(Directory dir, String id, String codecName,
- int codecVersion, Counter bytesUsed, IOContext context)
+ int codecVersion, Counter bytesUsed, IOContext context, Type type)
throws IOException {
this(dir, id, codecName, codecVersion, new DirectTrackingAllocator(
- ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, false);
+ ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, false, type);
}
protected DerefBytesWriterBase(Directory dir, String id, String codecName,
- int codecVersion, Counter bytesUsed, IOContext context, boolean fasterButMoreRam)
+ int codecVersion, Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type)
throws IOException {
this(dir, id, codecName, codecVersion, new DirectTrackingAllocator(
- ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, fasterButMoreRam);
+ ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, fasterButMoreRam,type);
}
protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Allocator allocator,
- Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException {
- super(dir, id, codecName, codecVersion, bytesUsed, context);
+ Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type) throws IOException {
+ super(dir, id, codecName, codecVersion, bytesUsed, context, type);
hash = new BytesRefHash(new ByteBlockPool(allocator),
BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
@@ -430,7 +416,9 @@
}
@Override
- protected void add(int docID, BytesRef bytes) throws IOException {
+ public void add(int docID, IndexableField value) throws IOException {
+ BytesRef bytes = value.binaryValue();
+ assert bytes != null;
if (bytes.length == 0) { // default value - skip it
return;
}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/BytesRefUtils.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/BytesRefUtils.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/BytesRefUtils.java (working copy)
@@ -1,120 +0,0 @@
-package org.apache.lucene.codecs.lucene40.values;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-import org.apache.lucene.util.BytesRef;
-
-/**
- * Package private BytesRefUtils - can move this into the o.a.l.utils package if
- * needed.
- *
- * @lucene.internal
- */
-final class BytesRefUtils {
-
- private BytesRefUtils() {
- }
-
- /**
- * Copies the given long value and encodes it as 8 byte Big-Endian.
- *
- * NOTE: this method resets the offset to 0, length to 8 and resizes the
- * reference array if needed.
- */
- public static void copyLong(BytesRef ref, long value) {
- if (ref.bytes.length < 8) {
- ref.bytes = new byte[8];
- }
- copyInternal(ref, (int) (value >> 32), ref.offset = 0);
- copyInternal(ref, (int) value, 4);
- ref.length = 8;
- }
-
- /**
- * Copies the given int value and encodes it as 4 byte Big-Endian.
- *
- * NOTE: this method resets the offset to 0, length to 4 and resizes the
- * reference array if needed.
- */
- public static void copyInt(BytesRef ref, int value) {
- if (ref.bytes.length < 4) {
- ref.bytes = new byte[4];
- }
- copyInternal(ref, value, ref.offset = 0);
- ref.length = 4;
- }
-
- /**
- * Copies the given short value and encodes it as a 2 byte Big-Endian.
- *
- * NOTE: this method resets the offset to 0, length to 2 and resizes the
- * reference array if needed.
- */
- public static void copyShort(BytesRef ref, short value) {
- if (ref.bytes.length < 2) {
- ref.bytes = new byte[2];
- }
- ref.bytes[ref.offset] = (byte) (value >> 8);
- ref.bytes[ref.offset + 1] = (byte) (value);
- ref.length = 2;
- }
-
- private static void copyInternal(BytesRef ref, int value, int startOffset) {
- ref.bytes[startOffset] = (byte) (value >> 24);
- ref.bytes[startOffset + 1] = (byte) (value >> 16);
- ref.bytes[startOffset + 2] = (byte) (value >> 8);
- ref.bytes[startOffset + 3] = (byte) (value);
- }
-
- /**
- * Converts 2 consecutive bytes from the current offset to a short. Bytes are
- * interpreted as Big-Endian (most significant bit first)
- *
- * NOTE: this method does NOT check the bounds of the referenced array.
- */
- public static short asShort(BytesRef b) {
- return (short) (0xFFFF & ((b.bytes[b.offset] & 0xFF) << 8) | (b.bytes[b.offset + 1] & 0xFF));
- }
-
- /**
- * Converts 4 consecutive bytes from the current offset to an int. Bytes are
- * interpreted as Big-Endian (most significant bit first)
- *
- * NOTE: this method does NOT check the bounds of the referenced array.
- */
- public static int asInt(BytesRef b) {
- return asIntInternal(b, b.offset);
- }
-
- /**
- * Converts 8 consecutive bytes from the current offset to a long. Bytes are
- * interpreted as Big-Endian (most significant bit first)
- *
- * NOTE: this method does NOT check the bounds of the referenced array.
- */
- public static long asLong(BytesRef b) {
- return (((long) asIntInternal(b, b.offset) << 32) | asIntInternal(b,
- b.offset + 4) & 0xFFFFFFFFL);
- }
-
- private static int asIntInternal(BytesRef b, int pos) {
- return ((b.bytes[pos++] & 0xFF) << 24) | ((b.bytes[pos++] & 0xFF) << 16)
- | ((b.bytes[pos++] & 0xFF) << 8) | (b.bytes[pos] & 0xFF);
- }
-
-}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesArray.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesArray.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesArray.java (working copy)
@@ -30,10 +30,12 @@
/**
* @lucene.experimental
+ * @lucene.internal
*/
-abstract class DocValuesArray extends Source {
+public abstract class DocValuesArray extends Source {
+ // TODO move to o.a.l.codecs
- static final Map TEMPLATES;
+ public static final Map TEMPLATES;
static {
EnumMap templates = new EnumMap(
@@ -54,8 +56,14 @@
this.bytesPerValue = bytesPerValue;
}
+ @Override
+ public abstract BytesRef getBytes(int docID, BytesRef ref);
+
+
public abstract DocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException;
+
+ public abstract DocValuesArray newFromArray(Object array);
@Override
public final boolean hasArray() {
@@ -63,20 +71,24 @@
}
void toBytes(long value, BytesRef bytesRef) {
- BytesRefUtils.copyLong(bytesRef, value);
+ copyLong(bytesRef, value);
}
void toBytes(double value, BytesRef bytesRef) {
- BytesRefUtils.copyLong(bytesRef, Double.doubleToRawLongBits(value));
+ copyLong(bytesRef, Double.doubleToRawLongBits(value));
}
final static class ByteValues extends DocValuesArray {
private final byte[] values;
-
+
ByteValues() {
super(1, Type.FIXED_INTS_8);
values = new byte[0];
}
+ private ByteValues(byte[] array) {
+ super(1, Type.FIXED_INTS_8);
+ values = array;
+ }
private ByteValues(IndexInput input, int numDocs) throws IOException {
super(1, Type.FIXED_INTS_8);
@@ -100,10 +112,27 @@
throws IOException {
return new ByteValues(input, numDocs);
}
+
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof byte[];
+ return new ByteValues((byte[]) array);
+ }
void toBytes(long value, BytesRef bytesRef) {
+ if (bytesRef.bytes.length == 0) {
+ bytesRef.bytes = new byte[1];
+ }
bytesRef.bytes[0] = (byte) (0xFFL & value);
+ bytesRef.offset = 0;
+ bytesRef.length = 1;
}
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getInt(docID), ref);
+ return ref;
+ }
};
@@ -114,6 +143,11 @@
super(RamUsageEstimator.NUM_BYTES_SHORT, Type.FIXED_INTS_16);
values = new short[0];
}
+
+ private ShortValues(short[] array) {
+ super(RamUsageEstimator.NUM_BYTES_SHORT, Type.FIXED_INTS_16);
+ values = array;
+ }
private ShortValues(IndexInput input, int numDocs) throws IOException {
super(RamUsageEstimator.NUM_BYTES_SHORT, Type.FIXED_INTS_16);
@@ -141,9 +175,21 @@
}
void toBytes(long value, BytesRef bytesRef) {
- BytesRefUtils.copyShort(bytesRef, (short) (0xFFFFL & value));
+ copyShort(bytesRef, (short) (0xFFFFL & value));
}
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof short[];
+ return new ShortValues((short[]) array);
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getInt(docID), ref);
+ return ref;
+ }
+
};
final static class IntValues extends DocValuesArray {
@@ -162,6 +208,11 @@
}
}
+ private IntValues(int[] array) {
+ super(RamUsageEstimator.NUM_BYTES_INT, Type.FIXED_INTS_32);
+ values = array;
+ }
+
@Override
public int[] getArray() {
return values;
@@ -180,9 +231,21 @@
}
void toBytes(long value, BytesRef bytesRef) {
- BytesRefUtils.copyInt(bytesRef, (int) (0xFFFFFFFF & value));
+ copyInt(bytesRef, (int) (0xFFFFFFFF & value));
}
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof int[];
+ return new IntValues((int[]) array);
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getInt(docID), ref);
+ return ref;
+ }
+
};
final static class LongValues extends DocValuesArray {
@@ -201,6 +264,11 @@
}
}
+ private LongValues(long[] array) {
+ super(RamUsageEstimator.NUM_BYTES_LONG, Type.FIXED_INTS_64);
+ values = array;
+ }
+
@Override
public long[] getArray() {
return values;
@@ -218,6 +286,18 @@
return new LongValues(input, numDocs);
}
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof long[];
+ return new LongValues((long[])array);
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getInt(docID), ref);
+ return ref;
+ }
+
};
final static class FloatValues extends DocValuesArray {
@@ -240,6 +320,11 @@
}
}
+ private FloatValues(float[] array) {
+ super(RamUsageEstimator.NUM_BYTES_FLOAT, Type.FLOAT_32);
+ values = array;
+ }
+
@Override
public float[] getArray() {
return values;
@@ -253,7 +338,7 @@
@Override
void toBytes(double value, BytesRef bytesRef) {
- BytesRefUtils.copyInt(bytesRef, Float.floatToRawIntBits((float)value));
+ copyInt(bytesRef, Float.floatToRawIntBits((float)value));
}
@@ -262,6 +347,18 @@
throws IOException {
return new FloatValues(input, numDocs);
}
+
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof float[];
+ return new FloatValues((float[]) array);
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getFloat(docID), ref);
+ return ref;
+ }
};
final static class DoubleValues extends DocValuesArray {
@@ -284,6 +381,11 @@
}
}
+ private DoubleValues(double[] array) {
+ super(RamUsageEstimator.NUM_BYTES_DOUBLE, Type.FLOAT_64);
+ values = array;
+ }
+
@Override
public double[] getArray() {
return values;
@@ -301,6 +403,108 @@
return new DoubleValues(input, numDocs);
}
+ @Override
+ public DocValuesArray newFromArray(Object array) {
+ assert array instanceof double[];
+ return new DoubleValues((double[]) array);
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ toBytes(getFloat(docID), ref);
+ return ref;
+ }
+
};
+
+ /**
+ * Copies the given long value and encodes it as 8 byte Big-Endian.
+ *
+ * NOTE: this method resets the offset to 0, length to 8 and resizes the
+ * reference array if needed.
+ */
+ public static void copyLong(BytesRef ref, long value) {
+ if (ref.bytes.length < 8) {
+ ref.bytes = new byte[8];
+ }
+ copyInternal(ref, (int) (value >> 32), ref.offset = 0);
+ copyInternal(ref, (int) value, 4);
+ ref.length = 8;
+ }
-}
+ /**
+ * Copies the given int value and encodes it as 4 byte Big-Endian.
+ *
+ * NOTE: this method resets the offset to 0, length to 4 and resizes the
+ * reference array if needed.
+ */
+ public static void copyInt(BytesRef ref, int value) {
+ if (ref.bytes.length < 4) {
+ ref.bytes = new byte[4];
+ }
+ copyInternal(ref, value, ref.offset = 0);
+ ref.length = 4;
+
+ }
+
+ /**
+ * Copies the given short value and encodes it as a 2 byte Big-Endian.
+ *
+ * NOTE: this method resets the offset to 0, length to 2 and resizes the
+ * reference array if needed.
+ */
+ public static void copyShort(BytesRef ref, short value) {
+ if (ref.bytes.length < 2) {
+ ref.bytes = new byte[2];
+ }
+ ref.offset = 0;
+ ref.bytes[ref.offset] = (byte) (value >> 8);
+ ref.bytes[ref.offset + 1] = (byte) (value);
+ ref.length = 2;
+ }
+
+ private static void copyInternal(BytesRef ref, int value, int startOffset) {
+ ref.bytes[startOffset] = (byte) (value >> 24);
+ ref.bytes[startOffset + 1] = (byte) (value >> 16);
+ ref.bytes[startOffset + 2] = (byte) (value >> 8);
+ ref.bytes[startOffset + 3] = (byte) (value);
+ }
+
+ /**
+ * Converts 2 consecutive bytes from the current offset to a short. Bytes are
+ * interpreted as Big-Endian (most significant bit first)
+ *
+ * NOTE: this method does NOT check the bounds of the referenced array.
+ */
+ public static short asShort(BytesRef b) {
+ return (short) (0xFFFF & ((b.bytes[b.offset] & 0xFF) << 8) | (b.bytes[b.offset + 1] & 0xFF));
+ }
+
+ /**
+ * Converts 4 consecutive bytes from the current offset to an int. Bytes are
+ * interpreted as Big-Endian (most significant bit first)
+ *
+ * NOTE: this method does NOT check the bounds of the referenced array.
+ */
+ public static int asInt(BytesRef b) {
+ return asIntInternal(b, b.offset);
+ }
+
+ /**
+ * Converts 8 consecutive bytes from the current offset to a long. Bytes are
+ * interpreted as Big-Endian (most significant bit first)
+ *
+ * NOTE: this method does NOT check the bounds of the referenced array.
+ */
+ public static long asLong(BytesRef b) {
+ return (((long) asIntInternal(b, b.offset) << 32) | asIntInternal(b,
+ b.offset + 4) & 0xFFFFFFFFL);
+ }
+
+ private static int asIntInternal(BytesRef b, int pos) {
+ return ((b.bytes[pos++] & 0xFF) << 24) | ((b.bytes[pos++] & 0xFF) << 16)
+ | ((b.bytes[pos++] & 0xFF) << 8) | (b.bytes[pos] & 0xFF);
+ }
+
+
+}
\ No newline at end of file
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java (working copy)
@@ -46,7 +46,7 @@
public static class Writer extends DerefBytesWriterBase {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_FIXED_DEREF);
}
@Override
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (working copy)
@@ -58,7 +58,7 @@
public Writer(Directory dir, String id, Comparator comp,
Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_FIXED_SORTED);
this.comp = comp;
}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java (working copy)
@@ -22,10 +22,12 @@
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesReaderBase;
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesSourceBase;
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesWriterBase;
+import org.apache.lucene.document.DocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -52,6 +54,7 @@
static final int VERSION_CURRENT = VERSION_START;
static abstract class FixedBytesWriterBase extends BytesWriterBase {
+ protected final DocValuesField bytesSpareField = new DocValuesField("", new BytesRef(), Type.BYTES_FIXED_STRAIGHT);
protected int lastDocID = -1;
// start at -1 if the first added value is > 0
protected int size = -1;
@@ -60,13 +63,20 @@
protected FixedBytesWriterBase(Directory dir, String id, String codecName,
int version, Counter bytesUsed, IOContext context) throws IOException {
- super(dir, id, codecName, version, bytesUsed, context);
+ this(dir, id, codecName, version, bytesUsed, context, Type.BYTES_FIXED_STRAIGHT);
+ }
+
+ protected FixedBytesWriterBase(Directory dir, String id, String codecName,
+ int version, Counter bytesUsed, IOContext context, Type type) throws IOException {
+ super(dir, id, codecName, version, bytesUsed, context, type);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
pool.nextBuffer();
}
@Override
- protected void add(int docID, BytesRef bytes) throws IOException {
+ public void add(int docID, IndexableField value) throws IOException {
+ final BytesRef bytes = value.binaryValue();
+ assert bytes != null;
assert lastDocID < docID;
if (size == -1) {
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java (working copy)
@@ -80,23 +80,20 @@
assert template != null;
}
- protected void add(int docID, double v) throws IOException {
- template.toBytes(v, bytesRef);
- add(docID, bytesRef);
- }
-
@Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().doubleValue());
- }
-
- @Override
protected boolean tryBulkMerge(DocValues docValues) {
// only bulk merge if value type is the same otherwise size differs
return super.tryBulkMerge(docValues) && docValues.type() == template.type();
}
@Override
+ public void add(int docID, IndexableField value) throws IOException {
+ template.toBytes(value.numericValue().doubleValue(), bytesRef);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
+ }
+
+ @Override
protected void setMergeBytes(Source source, int sourceDoc) {
final double value = source.getFloat(sourceDoc);
template.toBytes(value, bytesRef);
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java (working copy)
@@ -104,23 +104,20 @@
template = DocValuesArray.TEMPLATES.get(valueType);
}
- protected void add(int docID, long v) throws IOException {
- template.toBytes(v, bytesRef);
- add(docID, bytesRef);
- }
-
@Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().longValue());
- }
-
- @Override
protected void setMergeBytes(Source source, int sourceDoc) {
final long value = source.getInt(sourceDoc);
template.toBytes(value, bytesRef);
}
@Override
+ public void add(int docID, IndexableField value) throws IOException {
+ template.toBytes(value.numericValue().longValue(), bytesRef);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
+ }
+
+ @Override
protected boolean tryBulkMerge(DocValues docValues) {
// only bulk merge if value type is the same otherwise size differs
return super.tryBulkMerge(docValues) && docValues.type() == template.type();
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java (working copy)
@@ -20,6 +20,7 @@
import org.apache.lucene.codecs.lucene40.values.DocValuesArray.LongValues;
import org.apache.lucene.codecs.lucene40.values.FixedStraightBytesImpl.FixedBytesWriterBase;
+import org.apache.lucene.document.DocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
@@ -59,27 +60,10 @@
protected PackedIntsWriter(Directory dir, String id, Counter bytesUsed,
IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.VAR_INTS);
bytesRef = new BytesRef(8);
}
-
- protected void add(int docID, long v) throws IOException {
- assert lastDocId < docID;
- if (!started) {
- started = true;
- minValue = maxValue = v;
- } else {
- if (v < minValue) {
- minValue = v;
- } else if (v > maxValue) {
- maxValue = v;
- }
- }
- lastDocId = docID;
- BytesRefUtils.copyLong(bytesRef, v);
- add(docID, bytesRef);
- }
-
+
@Override
public void finish(int docCount) throws IOException {
boolean success = false;
@@ -112,13 +96,6 @@
}
}
- @Override
- protected void mergeDoc(Field scratchField, Source source, int docID, int sourceDoc) throws IOException {
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- add(docID, source.getInt(sourceDoc));
- }
-
private void writePackedInts(IndexOutput datOut, int docCount) throws IOException {
datOut.writeLong(minValue);
@@ -149,10 +126,25 @@
}
w.finish();
}
-
+
@Override
public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().longValue());
+ final long v = docValue.numericValue().longValue();
+ assert lastDocId < docID;
+ if (!started) {
+ started = true;
+ minValue = maxValue = v;
+ } else {
+ if (v < minValue) {
+ minValue = v;
+ } else if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ lastDocId = docID;
+ DocValuesArray.copyLong(bytesRef, v);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
}
}
@@ -247,7 +239,7 @@
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
ref.grow(8);
- BytesRefUtils.copyLong(ref, getInt(docID));
+ DocValuesArray.copyLong(ref, getInt(docID));
return ref;
}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java (working copy)
@@ -57,7 +57,7 @@
static class Writer extends DerefBytesWriterBase {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_DEREF);
size = 0;
}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (working copy)
@@ -59,7 +59,7 @@
public Writer(Directory dir, String id, Comparator comp,
Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_VAR_SORTED);
this.comp = comp;
size = 0;
}
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (working copy)
@@ -26,6 +26,7 @@
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -63,7 +64,7 @@
private boolean merge = false;
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_STRAIGHT);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
docToAddress = new long[1];
pool.nextBuffer(); // init
@@ -84,7 +85,9 @@
}
@Override
- protected void add(int docID, BytesRef bytes) throws IOException {
+ public void add(int docID, IndexableField value) throws IOException {
+ final BytesRef bytes = value.binaryValue();
+ assert bytes != null;
assert !merge;
if (bytes.length == 0) {
return; // default
Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (working copy)
@@ -40,6 +40,7 @@
*/
abstract class Writer extends DocValuesConsumer {
protected final Counter bytesUsed;
+ protected Type type;
/**
* Creates a new {@link Writer}.
@@ -49,10 +50,20 @@
* internally allocated memory. All tracked bytes must be released
* once {@link #finish(int)} has been called.
*/
- protected Writer(Counter bytesUsed) {
+ protected Writer(Counter bytesUsed, Type type) {
this.bytesUsed = bytesUsed;
+ this.type = type;
}
+
+
+ @Override
+ protected Type getType() {
+ return type;
+ }
+
+
+
/**
* Factory method to create a {@link Writer} instance for a given type. This
* method returns default implementations for each of the different types
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java (working copy)
@@ -26,7 +26,6 @@
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
/**
* plain text index format.
@@ -41,7 +40,7 @@
private final FieldInfosFormat fieldInfosFormat = new SimpleTextFieldInfosFormat();
private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat();
// TODO: need a plain-text impl
- private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
+ private final DocValuesFormat docValues = new SimpleTextDocValuesFormat();
// TODO: need a plain-text impl (using the above)
private final NormsFormat normsFormat = new SimpleTextNormsFormat();
private final LiveDocsFormat liveDocs = new SimpleTextLiveDocsFormat();
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java (working copy)
@@ -0,0 +1,288 @@
+package org.apache.lucene.codecs.simpletext;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.lucene40.values.DocValuesArray;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * @lucene.experimental
+ */
+public class SimpleTextDocValuesConsumer extends DocValuesConsumer {
+
+ static final BytesRef ZERO_DOUBLE = new BytesRef(Double.toString(0d));
+ static final BytesRef ZERO_INT = new BytesRef(Integer.toString(0));
+ static final BytesRef HEADER = new BytesRef("SimpleTextDocValues");
+
+ static final BytesRef END = new BytesRef("END");
+ static final BytesRef VALUE_SIZE = new BytesRef("valuesize ");
+ static final BytesRef DOC = new BytesRef(" doc ");
+ static final BytesRef VALUE = new BytesRef(" value ");
+ protected BytesRef scratch = new BytesRef();
+ protected int maxDocId = -1;
+ protected final String segment;
+ protected final Directory dir;
+ protected final IOContext ctx;
+ protected final Type type;
+ protected final BytesRefHash hash;
+ private int[] ords;
+ private int fixedSize = Integer.MIN_VALUE;
+ private BytesRef zeroBytes;
+ private final String segmentSuffix;
+
+
+ public SimpleTextDocValuesConsumer(String segment, Directory dir,
+ IOContext ctx, Type type, String segmentSuffix) {
+ this.ctx = ctx;
+ this.dir = dir;
+ this.segment = segment;
+ this.type = type;
+ hash = new BytesRefHash();
+ ords = new int[0];
+ this.segmentSuffix = segmentSuffix;
+
+ }
+
+ @Override
+ public void add(int docID, IndexableField value) throws IOException {
+ assert docID >= 0;
+ int ord = -1;
+ int vSize = -1;
+ switch (type) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ vSize = value.binaryValue().length;
+ ord = hash.add(value.binaryValue());
+ break;
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ vSize = -1;
+ try {
+ ord = hash.add(value.binaryValue());
+ } catch (NullPointerException e) {
+ System.err.println();
+ }
+ break;
+ case FIXED_INTS_16:
+ vSize = 2;
+ scratch.grow(2);
+ DocValuesArray.copyShort(scratch, value.numericValue().shortValue());
+ ord = hash.add(scratch);
+ break;
+ case FIXED_INTS_32:
+ vSize = 4;
+ scratch.grow(4);
+ DocValuesArray.copyInt(scratch, value.numericValue().intValue());
+ ord = hash.add(scratch);
+ break;
+ case FIXED_INTS_8:
+ vSize = 1;
+ scratch.grow(1);
+ scratch.bytes[scratch.offset] = value.numericValue().byteValue();
+ scratch.length = 1;
+ ord = hash.add(scratch);
+ break;
+ case FIXED_INTS_64:
+ vSize = 8;
+ case VAR_INTS:
+ scratch.grow(8);
+ DocValuesArray.copyLong(scratch, value.numericValue().longValue());
+ ord = hash.add(scratch);
+ break;
+ case FLOAT_32:
+ vSize = 4;
+ scratch.grow(4);
+ DocValuesArray.copyInt(scratch,
+ Float.floatToRawIntBits(value.numericValue().floatValue()));
+ ord = hash.add(scratch);
+ break;
+ case FLOAT_64:
+ vSize = 8;
+ scratch.grow(8);
+ DocValuesArray.copyLong(scratch,
+ Double.doubleToRawLongBits(value.numericValue().doubleValue()));
+ ord = hash.add(scratch);
+ break;
+
+ }
+
+ if (fixedSize == Integer.MIN_VALUE) {
+ assert maxDocId == -1;
+ fixedSize = vSize;
+ } else {
+ if (fixedSize != vSize) {
+ throw new IllegalArgumentException("value size must be " + fixedSize + " but was: " + vSize);
+ }
+ }
+ maxDocId = Math.max(docID, maxDocId);
+ ords = grow(ords, docID);
+
+ ords[docID] = (ord < 0 ? (-ord)-1 : ord) + 1;
+ }
+
+ protected BytesRef getHeader() {
+ return HEADER;
+ }
+
+ private int[] grow(int[] array, int upto) {
+ if (array.length <= upto) {
+ return ArrayUtil.grow(array, 1 + upto);
+ }
+ return array;
+ }
+
+ private void prepareFlush(int docCount) {
+ assert ords != null;
+ ords = grow(ords, docCount);
+ }
+
+ @Override
+ public void finish(int docCount) throws IOException {
+ final String fileName = IndexFileNames.segmentFileName(segment, "",
+ segmentSuffix);
+ IndexOutput output = dir.createOutput(fileName, ctx);
+ boolean success = false;
+ BytesRef spare = new BytesRef();
+ try {
+ SimpleTextUtil.write(output, getHeader());
+ SimpleTextUtil.writeNewline(output);
+ SimpleTextUtil.write(output, VALUE_SIZE);
+ SimpleTextUtil.write(output, Integer.toString(this.fixedSize), scratch);
+ SimpleTextUtil.writeNewline(output);
+ prepareFlush(docCount);
+ for (int i = 0; i < docCount; i++) {
+ SimpleTextUtil.write(output, DOC);
+ SimpleTextUtil.write(output, Integer.toString(i), scratch);
+ SimpleTextUtil.writeNewline(output);
+ SimpleTextUtil.write(output, VALUE);
+ writeDoc(output, i, spare);
+ SimpleTextUtil.writeNewline(output);
+ }
+ SimpleTextUtil.write(output, END);
+ SimpleTextUtil.writeNewline(output);
+ success = true;
+ } finally {
+ hash.close();
+ if (success) {
+ IOUtils.close(output);
+ } else {
+ IOUtils.closeWhileHandlingException(output);
+ }
+ }
+ }
+
+ protected void writeDoc(IndexOutput output, int docId, BytesRef spare) throws IOException {
+ int ord = ords[docId] - 1;
+ if (ord != -1) {
+ assert ord >= 0;
+ hash.get(ord, spare);
+
+ switch (type) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ SimpleTextUtil.write(output, spare);
+ break;
+ case FIXED_INTS_16:
+ SimpleTextUtil.write(output,
+ Short.toString(DocValuesArray.asShort(spare)), scratch);
+ break;
+ case FIXED_INTS_32:
+ SimpleTextUtil.write(output,
+ Integer.toString(DocValuesArray.asInt(spare)), scratch);
+ break;
+ case VAR_INTS:
+ case FIXED_INTS_64:
+ SimpleTextUtil.write(output,
+ Long.toString(DocValuesArray.asLong(spare)), scratch);
+ break;
+ case FIXED_INTS_8:
+ assert spare.length == 1 : spare.length;
+ SimpleTextUtil.write(output,
+ Integer.toString(spare.bytes[spare.offset]), scratch);
+ break;
+ case FLOAT_32:
+ float valueFloat = Float.intBitsToFloat(DocValuesArray.asInt(spare));
+ SimpleTextUtil.write(output, Float.toString(valueFloat), scratch);
+ break;
+ case FLOAT_64:
+ double valueDouble = Double.longBitsToDouble(DocValuesArray
+ .asLong(spare));
+ SimpleTextUtil.write(output, Double.toString(valueDouble), scratch);
+ break;
+ default:
+ throw new IllegalArgumentException("unsupported type: " + type);
+ }
+ } else {
+ switch (type) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ if(zeroBytes == null) {
+ assert fixedSize > 0;
+ zeroBytes = new BytesRef(new byte[fixedSize]);
+ }
+ SimpleTextUtil.write(output, zeroBytes);
+ break;
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ scratch.length = 0;
+ SimpleTextUtil.write(output, scratch);
+ break;
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case VAR_INTS:
+ SimpleTextUtil.write(output, ZERO_INT);
+ break;
+ case FLOAT_32:
+ case FLOAT_64:
+ SimpleTextUtil.write(output, ZERO_DOUBLE);
+ break;
+ default:
+ throw new IllegalArgumentException("unsupported type: " + type);
+ }
+ }
+
+ }
+
+ @Override
+ protected Type getType() {
+ return type;
+ }
+
+
+
+}
\ No newline at end of file
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java (working copy)
@@ -0,0 +1,53 @@
+package org.apache.lucene.codecs.simpletext;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.PerDocConsumer;
+import org.apache.lucene.codecs.PerDocProducer;
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.util.BytesRef;
+/**
+ * @lucene.experimental
+ */
+public class SimpleTextDocValuesFormat extends DocValuesFormat {
+ private static final String DOC_VALUES_SEG_SUFFIX = "dv";
+ @Override
+ public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+ return new SimpleTextPerDocConsumer(state, DOC_VALUES_SEG_SUFFIX);
+ }
+
+ @Override
+ public PerDocProducer docsProducer(SegmentReadState state) throws IOException {
+ return new SimpleTextPerDocProducer(state, BytesRef.getUTF8SortedAsUnicodeComparator(), DOC_VALUES_SEG_SUFFIX);
+ }
+
+ static String docValuesId(String segmentsName, int fieldId) {
+ return segmentsName + "_" + fieldId;
+ }
+
+ @Override
+ public void files(SegmentInfo info, Set files)
+ throws IOException {
+ SimpleTextPerDocConsumer.files(info, files, DOC_VALUES_SEG_SUFFIX);
+ }
+}
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsConsumer.java (working copy)
@@ -1,294 +0,0 @@
-package org.apache.lucene.codecs.simpletext;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.PerDocConsumer;
-import org.apache.lucene.index.DocValues.Type;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Writes plain-text norms
- *
- * FOR RECREATIONAL USE ONLY
- *
- * @lucene.experimental
- */
-public class SimpleTextNormsConsumer extends PerDocConsumer {
-
- /** Extension of norms file */
- static final String NORMS_EXTENSION = "len";
- final static BytesRef END = new BytesRef("END");
- final static BytesRef FIELD = new BytesRef("field ");
- final static BytesRef DOC = new BytesRef(" doc ");
- final static BytesRef NORM = new BytesRef(" norm ");
-
- private NormsWriter writer;
-
- private final Directory directory;
-
- private final String segment;
-
- private final IOContext context;
-
- public SimpleTextNormsConsumer(Directory directory, String segment,
- IOContext context) throws IOException {
- this.directory = directory;
- this.segment = segment;
- this.context = context;
- }
-
- @Override
- public void close() throws IOException {
- if (writer != null) {
- boolean success = false;
- try {
- writer.finish();
- success = true;
- } finally {
- if (success) {
- IOUtils.close(writer);
- } else {
- IOUtils.closeWhileHandlingException(writer);
- }
- }
- }
- }
-
- @Override
- protected DocValues getDocValuesForMerge(AtomicReader reader, FieldInfo info)
- throws IOException {
- return reader.normValues(info.name);
- }
-
- @Override
- protected boolean canMerge(FieldInfo info) {
- return info.normsPresent();
- }
-
- @Override
- protected Type getDocValuesType(FieldInfo info) {
- return info.getNormType();
- }
-
- @Override
- public DocValuesConsumer addValuesField(Type type, FieldInfo fieldInfo)
- throws IOException {
- if (type != Type.FIXED_INTS_8) {
- throw new UnsupportedOperationException("Codec only supports single byte norm values. Type give: " + type);
- }
- return new SimpleTextNormsDocValuesConsumer(fieldInfo);
- }
-
- @Override
- public void abort() {
- if (writer != null) {
- try {
- writer.abort();
- } catch (IOException e) {
- }
- }
- }
-
- private class SimpleTextNormsDocValuesConsumer extends DocValuesConsumer {
- // Holds all docID/norm pairs we've seen
- int[] docIDs = new int[1];
- byte[] norms = new byte[1];
- int upto;
- private final FieldInfo fi;
-
- public SimpleTextNormsDocValuesConsumer(FieldInfo fieldInfo) {
- fi = fieldInfo;
- }
-
- @Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().longValue());
- }
-
- public void add(int docID, long value) {
- if (docIDs.length <= upto) {
- assert docIDs.length == upto;
- docIDs = ArrayUtil.grow(docIDs, 1 + upto);
- }
- if (norms.length <= upto) {
- assert norms.length == upto;
- norms = ArrayUtil.grow(norms, 1 + upto);
- }
- norms[upto] = (byte) value;
-
- docIDs[upto] = docID;
- upto++;
- }
-
- @Override
- public void finish(int docCount) throws IOException {
- final NormsWriter normsWriter = getNormsWriter();
- boolean success = false;
- try {
- int uptoDoc = 0;
- normsWriter.setNumTotalDocs(docCount);
- if (upto > 0) {
- normsWriter.startField(fi);
- int docID = 0;
- for (; docID < docCount; docID++) {
- if (uptoDoc < upto && docIDs[uptoDoc] == docID) {
- normsWriter.writeNorm(norms[uptoDoc]);
- uptoDoc++;
- } else {
- normsWriter.writeNorm((byte) 0);
- }
- }
- // we should have consumed every norm
- assert uptoDoc == upto;
-
- } else {
- // Fill entire field with default norm:
- normsWriter.startField(fi);
- for (; upto < docCount; upto++)
- normsWriter.writeNorm((byte) 0);
- }
- success = true;
- } finally {
- if (!success) {
- normsWriter.abort();
- }
- }
- }
- }
-
- public NormsWriter getNormsWriter() throws IOException {
- if (writer == null) {
- writer = new NormsWriter(directory, segment, context);
- }
- return writer;
- }
-
- private static class NormsWriter implements Closeable{
-
- private final IndexOutput output;
- private int numTotalDocs = 0;
- private int docid = 0;
-
- private final BytesRef scratch = new BytesRef();
-
-
- public NormsWriter(Directory directory, String segment, IOContext context)
- throws IOException {
- final String normsFileName = IndexFileNames.segmentFileName(segment, "",
- NORMS_EXTENSION);
- output = directory.createOutput(normsFileName, context);
-
- }
-
- public void startField(FieldInfo info) throws IOException {
- assert info.omitNorms == false;
- docid = 0;
- write(FIELD);
- write(info.name);
- newLine();
- }
-
- public void writeNorm(byte norm) throws IOException {
- write(DOC);
- write(Integer.toString(docid));
- newLine();
-
- write(NORM);
- write(norm);
- newLine();
- docid++;
- }
-
- public void finish(int numDocs) throws IOException {
- if (docid != numDocs) {
- throw new RuntimeException(
- "mergeNorms produced an invalid result: docCount is " + numDocs
- + " but only saw " + docid + " file=" + output.toString()
- + "; now aborting this merge to prevent index corruption");
- }
- write(END);
- newLine();
- }
-
- private void write(String s) throws IOException {
- SimpleTextUtil.write(output, s, scratch);
- }
-
- private void write(BytesRef bytes) throws IOException {
- SimpleTextUtil.write(output, bytes);
- }
-
- private void write(byte b) throws IOException {
- scratch.grow(1);
- scratch.bytes[scratch.offset] = b;
- scratch.length = 1;
- SimpleTextUtil.write(output, scratch);
- }
-
- private void newLine() throws IOException {
- SimpleTextUtil.writeNewline(output);
- }
-
- public void setNumTotalDocs(int numTotalDocs) {
- assert this.numTotalDocs == 0 || numTotalDocs == this.numTotalDocs;
- this.numTotalDocs = numTotalDocs;
- }
-
- public void abort() throws IOException {
- close();
- }
-
- public void finish() throws IOException {
- finish(numTotalDocs);
- }
-
- @Override
- public void close() throws IOException {
- output.close();
- }
- }
-
- public static void files(SegmentInfo info, Set files) throws IOException {
- FieldInfos fieldInfos = info.getFieldInfos();
-
- for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.normsPresent()) {
- files.add(IndexFileNames.segmentFileName(info.name, "",
- NORMS_EXTENSION));
- break;
- }
- }
- }
-}
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (working copy)
@@ -18,35 +18,123 @@
*/
import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.PerDocProducer;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* plain-text norms format
*
* FOR RECREATIONAL USE ONLY
+ *
* @lucene.experimental
*/
public class SimpleTextNormsFormat extends NormsFormat {
+ private static final String NORMS_SEG_SUFFIX = "len";
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new SimpleTextNormsConsumer(state.directory, state.segmentName, state.context);
+ return new SimpleTextNormsPerDocConsumer(state, NORMS_SEG_SUFFIX);
}
-
+
@Override
public PerDocProducer docsProducer(SegmentReadState state) throws IOException {
- return new SimpleTextNormsProducer(state.dir, state.segmentInfo, state.fieldInfos, state.context);
+ return new SimpleTextNormsPerDocProducer(state,
+ BytesRef.getUTF8SortedAsUnicodeComparator(), NORMS_SEG_SUFFIX);
}
-
+
@Override
public void files(SegmentInfo info, Set files) throws IOException {
- SimpleTextNormsConsumer.files(info, files);
- }
+ SimpleTextNormsPerDocConsumer.files(info, files);
+ }
+
+ public static class SimpleTextNormsPerDocProducer extends
+ SimpleTextPerDocProducer {
+
+ public SimpleTextNormsPerDocProducer(SegmentReadState state,
+ Comparator comp, String segmentSuffix) throws IOException {
+ super(state, comp, segmentSuffix);
+ }
+
+ @Override
+ protected boolean canLoad(FieldInfo info) {
+ return info.normsPresent();
+ }
+
+ @Override
+ protected Type getDocValuesType(FieldInfo info) {
+ return info.getNormType();
+ }
+
+ @Override
+ protected boolean anyDocValuesFields(FieldInfos infos) {
+ return infos.hasNorms();
+ }
+
+ }
+
+ public static class SimpleTextNormsPerDocConsumer extends
+ SimpleTextPerDocConsumer {
+
+ public SimpleTextNormsPerDocConsumer(PerDocWriteState state,
+ String segmentSuffix) throws IOException {
+ super(state, segmentSuffix);
+ }
+
+ @Override
+ protected DocValues getDocValuesForMerge(AtomicReader reader, FieldInfo info)
+ throws IOException {
+ return reader.normValues(info.name);
+ }
+
+ @Override
+ protected boolean canMerge(FieldInfo info) {
+ return info.normsPresent();
+ }
+
+ @Override
+ protected Type getDocValuesType(FieldInfo info) {
+ return info.getNormType();
+ }
+
+ @Override
+ public void abort() {
+ Set files = new HashSet();
+ filesInternal(state.fieldInfos, state.segmentName, files, segmentSuffix);
+ IOUtils.deleteFilesIgnoringExceptions(state.directory,
+ files.toArray(new String[0]));
+ }
+
+ public static void files(SegmentInfo segmentInfo, Set files)
+ throws IOException {
+ filesInternal(segmentInfo.getFieldInfos(), segmentInfo.name, files,
+ NORMS_SEG_SUFFIX);
+ }
+
+ public static void filesInternal(FieldInfos fieldInfos, String segmentName,
+ Set files, String segmentSuffix) {
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.normsPresent()) {
+ String id = docValuesId(segmentName, fieldInfo.number);
+ files.add(IndexFileNames.segmentFileName(id, "",
+ segmentSuffix));
+ }
+ }
+ }
+ }
}
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsProducer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsProducer.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsProducer.java (working copy)
@@ -1,175 +0,0 @@
-package org.apache.lucene.codecs.simpletext;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.apache.lucene.codecs.simpletext.SimpleTextNormsConsumer.DOC;
-import static org.apache.lucene.codecs.simpletext.SimpleTextNormsConsumer.END;
-import static org.apache.lucene.codecs.simpletext.SimpleTextNormsConsumer.FIELD;
-import static org.apache.lucene.codecs.simpletext.SimpleTextNormsConsumer.NORM;
-import static org.apache.lucene.codecs.simpletext.SimpleTextNormsConsumer.NORMS_EXTENSION;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.codecs.PerDocProducer;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.DocValues.Type;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
-
-/**
- * Reads plain-text norms
- *
- * FOR RECREATIONAL USE ONLY
- * @lucene.experimental
- */
-public class SimpleTextNormsProducer extends PerDocProducer {
-
- Map norms = new HashMap();
-
- public SimpleTextNormsProducer(Directory directory, SegmentInfo si, FieldInfos fields, IOContext context) throws IOException {
- if (fields.hasNorms()) {
- readNorms(directory.openInput(IndexFileNames.segmentFileName(si.name, "", NORMS_EXTENSION), context), si.docCount);
- }
- }
-
- // we read in all the norms up front into a hashmap
- private void readNorms(IndexInput in, int maxDoc) throws IOException {
- BytesRef scratch = new BytesRef();
- boolean success = false;
- try {
- SimpleTextUtil.readLine(in, scratch);
- while (!scratch.equals(END)) {
- assert StringHelper.startsWith(scratch, FIELD);
- final String fieldName = readString(FIELD.length, scratch);
- byte bytes[] = new byte[maxDoc];
- for (int i = 0; i < bytes.length; i++) {
- SimpleTextUtil.readLine(in, scratch);
- assert StringHelper.startsWith(scratch, DOC);
- SimpleTextUtil.readLine(in, scratch);
- assert StringHelper.startsWith(scratch, NORM);
- bytes[i] = scratch.bytes[scratch.offset + NORM.length];
- }
- norms.put(fieldName, new NormsDocValues(new Norm(bytes)));
- SimpleTextUtil.readLine(in, scratch);
- assert StringHelper.startsWith(scratch, FIELD) || scratch.equals(END);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- norms = null;
- }
-
- static void files(Directory dir, SegmentInfo info, Set files) throws IOException {
- FieldInfos fieldInfos = info.getFieldInfos();
- for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.normsPresent()) {
- files.add(IndexFileNames.segmentFileName(info.name, "", SimpleTextNormsConsumer.NORMS_EXTENSION));
- break;
- }
- }
- }
-
- private String readString(int offset, BytesRef scratch) {
- return new String(scratch.bytes, scratch.offset+offset, scratch.length-offset, IOUtils.CHARSET_UTF_8);
- }
-
- @Override
- public DocValues docValues(String field) throws IOException {
- return norms.get(field);
- }
-
- private class NormsDocValues extends DocValues {
- private final Source source;
- public NormsDocValues(Source source) {
- this.source = source;
- }
-
- @Override
- public Source load() throws IOException {
- return source;
- }
-
- @Override
- public Source getDirectSource() throws IOException {
- return getSource();
- }
-
- @Override
- public Type type() {
- return Type.FIXED_INTS_8;
- }
-
- @Override
- public int getValueSize() {
- return 1;
- }
- }
-
- static final class Norm extends Source {
- protected Norm(byte[] bytes) {
- super(Type.FIXED_INTS_8);
- this.bytes = bytes;
- }
- final byte bytes[];
-
- @Override
- public BytesRef getBytes(int docID, BytesRef ref) {
- ref.bytes = bytes;
- ref.offset = docID;
- ref.length = 1;
- return ref;
- }
-
- @Override
- public long getInt(int docID) {
- return bytes[docID];
- }
-
- @Override
- public boolean hasArray() {
- return true;
- }
-
- @Override
- public Object getArray() {
- return bytes;
- }
-
- }
-}
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocConsumer.java (working copy)
@@ -0,0 +1,94 @@
+package org.apache.lucene.codecs.simpletext;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.PerDocConsumer;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * @lucene.experimental
+ */
+class SimpleTextPerDocConsumer extends PerDocConsumer {
+
+ protected final PerDocWriteState state;
+ protected final String segmentSuffix;
+ public SimpleTextPerDocConsumer(PerDocWriteState state, String segmentSuffix)
+ throws IOException {
+ this.state = state;
+ this.segmentSuffix = segmentSuffix;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public DocValuesConsumer addValuesField(Type type, FieldInfo field)
+ throws IOException {
+ return new SimpleTextDocValuesConsumer(SimpleTextDocValuesFormat.docValuesId(state.segmentName,
+ field.number), state.directory, state.context, type, segmentSuffix);
+ }
+
+ @Override
+ public void abort() {
+ Set files = new HashSet();
+ files(state.directory, state.fieldInfos, state.segmentName, files, segmentSuffix);
+ IOUtils.deleteFilesIgnoringExceptions(state.directory,
+ files.toArray(new String[0]));
+ }
+
+
+ static void files(SegmentInfo info, Set files, String segmentSuffix) throws IOException {
+ files(info.dir, info.getFieldInfos(), info.name, files, segmentSuffix);
+ }
+
+ static String docValuesId(String segmentsName, int fieldId) {
+ return segmentsName + "_" + fieldId;
+ }
+
+ @SuppressWarnings("fallthrough")
+ private static void files(Directory dir, FieldInfos fieldInfos,
+ String segmentName, Set files, String segmentSuffix) {
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.hasDocValues()) {
+ String filename = docValuesId(segmentName, fieldInfo.number);
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ segmentSuffix));
+ try {
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ segmentSuffix));
+ } catch (IOException e) {
+ // don't throw checked exception - dir is only used in assert
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java (working copy)
@@ -0,0 +1,431 @@
+package org.apache.lucene.codecs.simpletext;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesConsumer.DOC;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesConsumer.END;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesConsumer.HEADER;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesConsumer.VALUE;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesConsumer.VALUE_SIZE;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.lucene.codecs.lucene40.values.DocValuesArray;
+import org.apache.lucene.codecs.lucene40.values.DocValuesReaderBase;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.SortedSource;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.packed.PackedInts.Reader;
+
+/**
+ * @lucene.experimental
+ */
+public class SimpleTextPerDocProducer extends DocValuesReaderBase {
+ protected final TreeMap docValues;
+ private Comparator comp;
+ private final String segmentSuffix;
+
+ /**
+ * Creates a new {@link Lucene40DocValuesProducer} instance and loads all
+ * {@link DocValues} instances for this segment and codec.
+ */
+ public SimpleTextPerDocProducer(SegmentReadState state,
+ Comparator comp, String segmentSuffix) throws IOException {
+ this.comp = comp;
+ this.segmentSuffix = segmentSuffix;
+ if (anyDocValuesFields(state.fieldInfos)) {
+ docValues = load(state.fieldInfos, state.segmentInfo.name,
+ state.segmentInfo.docCount, state.dir, state.context);
+ } else {
+ docValues = new TreeMap();
+ }
+ }
+
+ @Override
+ protected Map docValues() {
+ return docValues;
+ }
+
+ protected DocValues loadDocValues(int docCount, Directory dir, String id,
+ DocValues.Type type, IOContext context) throws IOException {
+ return new SimpleTextDocValues(dir, context, type, id, docCount, comp, segmentSuffix);
+ }
+
+ @Override
+ protected void closeInternal(Collection closeables)
+ throws IOException {
+ IOUtils.close(closeables);
+ }
+
+ private static class SimpleTextDocValues extends DocValues {
+
+ private int docCount;
+
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ IOUtils.close(input);
+ }
+ }
+
+ private Type type;
+ private Comparator comp;
+ private int valueSize;
+ private final IndexInput input;
+
+ public SimpleTextDocValues(Directory dir, IOContext ctx, Type type,
+ String id, int docCount, Comparator comp, String segmentSuffix) throws IOException {
+ this.type = type;
+ this.docCount = docCount;
+ this.comp = comp;
+ final String fileName = IndexFileNames.segmentFileName(id, "", segmentSuffix);
+ boolean success = false;
+ IndexInput in = null;
+ try {
+ in = dir.openInput(fileName, ctx);
+ valueSize = readHeader(in);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+ input = in;
+
+ }
+
+ @Override
+ public Source load() throws IOException {
+ boolean success = false;
+ IndexInput in = (IndexInput) input.clone();
+ try {
+ Source source = null;
+ switch (type) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ source = read(in, new ValueReader(type, docCount, comp));
+ break;
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case VAR_INTS:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case FLOAT_32:
+ case FLOAT_64:
+ source = read(in, new ValueReader(type, docCount, null));
+ break;
+ default:
+ throw new IllegalArgumentException("unknown type: " + type);
+ }
+ assert source != null;
+ success = true;
+ return source;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(in);
+ } else {
+ IOUtils.close(in);
+ }
+ }
+ }
+
+ private int readHeader(IndexInput in) throws IOException {
+ BytesRef scratch = new BytesRef();
+ SimpleTextUtil.readLine(in, scratch);
+ assert StringHelper.startsWith(scratch, HEADER);
+ SimpleTextUtil.readLine(in, scratch);
+ assert StringHelper.startsWith(scratch, VALUE_SIZE);
+ return Integer.parseInt(readString(scratch.offset + VALUE_SIZE.length,
+ scratch));
+ }
+
+ private Source read(IndexInput in, ValueReader reader) throws IOException {
+ BytesRef scratch = new BytesRef();
+ for (int i = 0; i < docCount; i++) {
+ SimpleTextUtil.readLine(in, scratch);
+
+ assert StringHelper.startsWith(scratch, DOC) : scratch.utf8ToString();
+ SimpleTextUtil.readLine(in, scratch);
+ assert StringHelper.startsWith(scratch, VALUE);
+ reader.fromString(i, scratch, scratch.offset + VALUE.length);
+ }
+ SimpleTextUtil.readLine(in, scratch);
+ assert scratch.equals(END);
+ return reader.getSource();
+ }
+
+ @Override
+ public Source getDirectSource() throws IOException {
+ return this.getSource();
+ }
+
+ @Override
+ public int getValueSize() {
+ return valueSize;
+ }
+
+ @Override
+ public Type type() {
+ return type;
+ }
+
+ }
+
+ public static String readString(int offset, BytesRef scratch) {
+ return new String(scratch.bytes, scratch.offset + offset, scratch.length
+ - offset, IOUtils.CHARSET_UTF_8);
+ }
+
+ private static final class ValueReader {
+ private final Type type;
+ private byte[] bytes;
+ private short[] shorts;
+ private int[] ints;
+ private long[] longs;
+ private float[] floats;
+ private double[] doubles;
+ private Source source;
+ private BytesRefHash hash;
+ private BytesRef scratch;
+
+ public ValueReader(Type type, int maxDocs, Comparator comp) {
+ super();
+ this.type = type;
+ Source docValuesArray = null;
+ switch (type) {
+ case FIXED_INTS_16:
+ shorts = new short[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type)
+ .newFromArray(shorts);
+ break;
+ case FIXED_INTS_32:
+ ints = new int[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type).newFromArray(ints);
+ break;
+ case FIXED_INTS_64:
+ longs = new long[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type)
+ .newFromArray(longs);
+ break;
+ case VAR_INTS:
+ longs = new long[maxDocs];
+ docValuesArray = new VarIntsArraySource(type, longs);
+ break;
+ case FIXED_INTS_8:
+ bytes = new byte[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type).newFromArray(bytes);
+ break;
+ case FLOAT_32:
+ floats = new float[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type)
+ .newFromArray(floats);
+ break;
+ case FLOAT_64:
+ doubles = new double[maxDocs];
+ docValuesArray = DocValuesArray.TEMPLATES.get(type).newFromArray(
+ doubles);
+ break;
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ assert comp != null;
+ hash = new BytesRefHash();
+ BytesSource bytesSource = new BytesSource(type, comp, maxDocs, hash);
+ ints = bytesSource.docIdToEntry;
+ source = bytesSource;
+ scratch = new BytesRef();
+ break;
+
+ }
+ if (docValuesArray != null) {
+ assert source == null;
+ this.source = docValuesArray;
+ }
+ }
+
+ public void fromString(int ord, BytesRef ref, int offset) {
+ switch (type) {
+ case FIXED_INTS_16:
+ assert shorts != null;
+ shorts[ord] = Short.parseShort(readString(offset, ref));
+ break;
+ case FIXED_INTS_32:
+ assert ints != null;
+ ints[ord] = Integer.parseInt(readString(offset, ref));
+ break;
+ case FIXED_INTS_64:
+ case VAR_INTS:
+ assert longs != null;
+ longs[ord] = Long.parseLong(readString(offset, ref));
+ break;
+ case FIXED_INTS_8:
+ assert bytes != null;
+ bytes[ord] = (byte) Integer.parseInt(readString(offset, ref));
+ break;
+ case FLOAT_32:
+ assert floats != null;
+ floats[ord] = Float.parseFloat(readString(offset, ref));
+ break;
+ case FLOAT_64:
+ assert doubles != null;
+ doubles[ord] = Double.parseDouble(readString(offset, ref));
+ break;
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ scratch.bytes = ref.bytes;
+ scratch.length = ref.length - offset;
+ scratch.offset = ref.offset + offset;
+ int key = hash.add(scratch);
+ ints[ord] = key < 0 ? (-key) - 1 : key;
+ break;
+ }
+ }
+
+ public Source getSource() {
+ if (source instanceof BytesSource) {
+ ((BytesSource) source).maybeSort();
+ }
+ return source;
+ }
+ }
+
+ private static final class BytesSource extends SortedSource {
+
+ private final BytesRefHash hash;
+ int[] docIdToEntry;
+ int[] sortedEntries;
+ int[] adresses;
+ private final boolean isSorted;
+
+ protected BytesSource(Type type, Comparator comp, int maxDoc,
+ BytesRefHash hash) {
+ super(type, comp);
+ docIdToEntry = new int[maxDoc];
+ this.hash = hash;
+ isSorted = type == Type.BYTES_FIXED_SORTED
+ || type == Type.BYTES_VAR_SORTED;
+ }
+
+ void maybeSort() {
+ if (isSorted) {
+ adresses = new int[hash.size()];
+ sortedEntries = hash.sort(getComparator());
+ for (int i = 0; i < adresses.length; i++) {
+ int entry = sortedEntries[i];
+ adresses[entry] = i;
+ }
+ }
+
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ if (isSorted) {
+ return hash.get(sortedEntries[ord(docID)], ref);
+ } else {
+ return hash.get(docIdToEntry[docID], ref);
+ }
+ }
+
+ @Override
+ public SortedSource asSortedSource() {
+ if (isSorted) {
+ return this;
+ }
+ return null;
+ }
+
+ @Override
+ public int ord(int docID) {
+ assert isSorted;
+ try {
+ return adresses[docIdToEntry[docID]];
+ } catch (Exception e) {
+
+ return 0;
+ }
+ }
+
+ @Override
+ public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+ assert isSorted;
+ return hash.get(sortedEntries[ord], bytesRef);
+ }
+
+ @Override
+ public Reader getDocToOrd() {
+ return null;
+ }
+
+ @Override
+ public int getValueCount() {
+ return hash.size();
+ }
+
+ }
+
+ private static class VarIntsArraySource extends Source {
+
+ private final long[] array;
+
+ protected VarIntsArraySource(Type type, long[] array) {
+ super(type);
+ this.array = array;
+ }
+
+ @Override
+ public long getInt(int docID) {
+ return array[docID];
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ DocValuesArray.copyLong(ref, getInt(docID));
+ return ref;
+ }
+
+ }
+
+}
Index: lucene/core/src/java/org/apache/lucene/index/IndexableField.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexableField.java (revision 1296264)
+++ lucene/core/src/java/org/apache/lucene/index/IndexableField.java (working copy)
@@ -54,7 +54,7 @@
/** Non-null if this field has a Reader value */
public Reader readerValue();
- /** Non-null if this field hasa numeric value */
+ /** Non-null if this field has a numeric value */
public Number numericValue();
/**
Index: lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (revision 1296264)
+++ lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (working copy)
@@ -148,8 +148,8 @@
Directory target = newDirectory();
IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean()));
- IndexReader r_1 = IndexReader.open(w_1, true);
- IndexReader r_2 = IndexReader.open(w_2, true);
+ DirectoryReader r_1 = DirectoryReader.open(w_1, true);
+ DirectoryReader r_2 = DirectoryReader.open(w_2, true);
if (random.nextBoolean()) {
w.addIndexes(d_1, d_2);
} else {
@@ -163,7 +163,7 @@
// check values
- IndexReader merged = IndexReader.open(w, true);
+ DirectoryReader merged = DirectoryReader.open(w, true);
Source source_1 = getSource(getDocValues(r_1, first.name()));
Source source_2 = getSource(getDocValues(r_2, second.name()));
Source source_1_merged = getSource(getDocValues(merged, first.name()));
@@ -260,7 +260,7 @@
FixedBitSet deleted = indexValues(w, numValues, val, numVariantList,
withDeletions, 7);
List closeables = new ArrayList();
- IndexReader r = IndexReader.open(w, true);
+ DirectoryReader r = DirectoryReader.open(w, true);
final int numRemainingValues = numValues - deleted.cardinality();
final int base = r.numDocs() - numRemainingValues;
// for FIXED_INTS_8 we use value mod 128 - to enable testing in
@@ -338,7 +338,7 @@
final int bytesSize = 1 + atLeast(50);
FixedBitSet deleted = indexValues(w, numValues, byteIndexValue,
byteVariantList, withDeletions, bytesSize);
- final IndexReader r = IndexReader.open(w, withDeletions);
+ final DirectoryReader r = DirectoryReader.open(w, withDeletions);
assertEquals(0, r.numDeletedDocs());
final int numRemainingValues = numValues - deleted.cardinality();
final int base = r.numDocs() - numRemainingValues;
@@ -422,12 +422,16 @@
for (Type val : numVariantList) {
indexValues(w, numValues, val, numVariantList,
false, 7);
- IndexReader r = IndexReader.open(w, true);
+ DirectoryReader r = DirectoryReader.open(w, true);
+ if (val == Type.VAR_INTS) {
+ DocValues docValues = getDocValues(r, val.name());
+ }
DocValues docValues = getDocValues(r, val.name());
assertNotNull(docValues);
// make sure we don't get a direct source since they don't support getArray()
+ if (val == Type.VAR_INTS) {
+ }
Source source = docValues.getSource();
-
switch (source.type()) {
case FIXED_INTS_8:
{
@@ -465,7 +469,8 @@
}
}
break;
- case VAR_INTS:
+ case VAR_INTS:
+ System.out.println(source.hasArray());
assertFalse(source.hasArray());
break;
case FLOAT_32:
@@ -503,7 +508,7 @@
final int numValues = 50 + atLeast(10);
// only single byte fixed straight supports getArray()
indexValues(w, numValues, Type.BYTES_FIXED_STRAIGHT, null, false, 1);
- IndexReader r = IndexReader.open(w, true);
+ DirectoryReader r = DirectoryReader.open(w, true);
DocValues docValues = getDocValues(r, Type.BYTES_FIXED_STRAIGHT.name());
assertNotNull(docValues);
// make sure we don't get a direct source since they don't support
@@ -513,12 +518,13 @@
switch (source.type()) {
case BYTES_FIXED_STRAIGHT: {
BytesRef ref = new BytesRef();
- assertTrue(source.hasArray());
- byte[] values = (byte[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- source.getBytes(i, ref);
- assertEquals(1, ref.length);
- assertEquals(values[i], ref.bytes[ref.offset]);
+ if (source.hasArray()) {
+ byte[] values = (byte[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ source.getBytes(i, ref);
+ assertEquals(1, ref.length);
+ assertEquals(values[i], ref.bytes[ref.offset]);
+ }
}
}
break;
@@ -925,4 +931,4 @@
r.close();
dir.close();
}
-}
+}
\ No newline at end of file
Index: lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java (revision 1296264)
+++ lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWNormsConsumer.java (working copy)
@@ -163,6 +163,11 @@
docIDs[upto] = docID;
upto++;
}
+
+ @Override
+ protected Type getType() {
+ return Type.FIXED_INTS_8;
+ }
}