Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy)
@@ -992,6 +992,10 @@
values.getFloat();
break;
case INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
values.getInt();
break;
default:
Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy)
@@ -636,6 +636,19 @@
case BYTES_VAR_SORTED:
b = 9;
break;
+ case FIXED_INTS_16:
+ b = 10;
+ break;
+ case FIXED_INTS_32:
+ b = 11;
+ break;
+ case FIXED_INTS_64:
+ b = 12;
+ break;
+ case FIXED_INTS_8:
+ b = 13;
+ break;
+
default:
throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
}
@@ -712,6 +725,19 @@
case 9:
docValuesType = ValueType.BYTES_VAR_SORTED;
break;
+ case 10:
+ docValuesType = ValueType.FIXED_INTS_16;
+ break;
+ case 11:
+ docValuesType = ValueType.FIXED_INTS_32;
+ break;
+ case 12:
+ docValuesType = ValueType.FIXED_INTS_64;
+ break;
+ case 13:
+ docValuesType = ValueType.FIXED_INTS_8;
+ break;
+
default:
throw new IllegalStateException("unhandled indexValues type " + b);
}
Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (working copy)
@@ -81,11 +81,16 @@
case FLOAT_32:
case FLOAT_64:
case INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
files.add(IndexFileNames.segmentFileName(filename, "",
Writer.DATA_EXTENSION));
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
Writer.DATA_EXTENSION));
break;
+
default:
assert false;
}
Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (working copy)
@@ -121,6 +121,10 @@
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
ValueType type) throws IOException {
switch (type) {
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
case INTS:
return Ints.getValues(dir, id, false);
case FLOAT_32:
Index: lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (working copy)
@@ -254,6 +254,25 @@
*/
public abstract ValuesEnum getEnum(AttributeSource attrSource)
throws IOException;
+
+ /**
+ * Returns true iff this {@link Source} exposes an array via
+ * {@link #getArray()} otherwise false.
+ *
+ * @return true iff this {@link Source} exposes an array via
+ * {@link #getArray()} otherwise false.
+ */
+ public boolean hasArray() {
+ return false;
+ }
+
+ /**
+ * Returns the internal array representation iff this {@link Source} uses an
+ * array as its inner representation, otherwise null.
+ */
+ public Object getArray() {
+ return null;
+ }
}
/**
Index: lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java (revision 0)
+++ lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java (revision 0)
@@ -0,0 +1,511 @@
+package org.apache.lucene.index.values;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.index.values.IndexDocValues.Source;
+import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * @lucene.experimental
+ */
+abstract class IndexDocValuesArray extends Source {
+
+ private final AtomicLong bytesUsed;
+ private final int bytesPerValue;
+ protected int maxDocID = -1;
+ private int currentSize = 0;
+ protected final ValueType type;
+
+ IndexDocValuesArray(AtomicLong bytesUsed, int bytesPerValue, ValueType type) {
+ this.bytesUsed = bytesUsed;
+ this.bytesPerValue = bytesPerValue;
+ this.type = type;
+ }
+
+ IndexDocValuesArray(ValueType type) {
+ this(null, 0, type);
+ }
+
+ void set(int docId, long value) {
+ if (docId >= currentSize) {
+ adjustSize(grow(docId + 1));
+ }
+ maxDocID = Math.max(docId, maxDocID);
+ setInternal(docId, value);
+ }
+
+ protected final void adjustSize(int newSize) {
+ if (bytesUsed != null) {
+ bytesUsed.addAndGet(bytesPerValue * (newSize - currentSize));
+ }
+ currentSize = newSize;
+ }
+
+ void clear() {
+ adjustSize(0);
+ maxDocID = -1;
+ currentSize = 0;
+ }
+
+ protected abstract void setInternal(int docId, long value);
+
+ protected abstract int grow(int numDocs);
+
+ abstract void read(IndexInput input, int numDocs) throws IOException;
+
+ abstract void write(IndexOutput output, int numDocs) throws IOException;
+
+ @Override
+ public int getValueCount() {
+ return maxDocID + 1;
+ }
+
+ @Override
+ public ValueType type() {
+ return type;
+ }
+
+ @Override
+ public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+ return new SourceEnum(attrSource, type(), this, maxDocID + 1) {
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= numDocs) {
+ return pos = NO_MORE_DOCS;
+ }
+ intsRef.ints[intsRef.offset] = IndexDocValuesArray.this.getInt(target);
+ return pos = target;
+ }
+ };
+ }
+
+ abstract ValuesEnum getDirectEnum(AttributeSource attrSource,
+ IndexInput input) throws IOException;
+
+ @Override
+ public final boolean hasArray() {
+ return true;
+ }
+
+
+ static IndexDocValuesArray forType(ValueType type) {
+ return forType(type, null);
+ }
+
+ static IndexDocValuesArray forType(ValueType type, AtomicLong bytesUsed) {
+ switch (type) {
+ case FIXED_INTS_16:
+ return new ShortValues(bytesUsed);
+ case FIXED_INTS_32:
+ return new IntValues(bytesUsed);
+ case FIXED_INTS_64:
+ return new LongValues(bytesUsed);
+ case FIXED_INTS_8:
+ return new ByteValues(bytesUsed);
+ case INTS:
+ return new LongValues(bytesUsed);
+ default:
+ return null;
+ }
+ }
+
+ final static class ByteValues extends IndexDocValuesArray {
+ private byte[] values;
+
+ ByteValues(AtomicLong bytesUsed) {
+ super(bytesUsed, 1, ValueType.FIXED_INTS_8);
+ values = new byte[0];
+ }
+
+ ByteValues() {
+ super(ValueType.FIXED_INTS_8);
+ }
+
+ @Override
+ public byte[] getArray() {
+ return values;
+ }
+
+ @Override
+ public long getInt(int docID) {
+ assert docID >= 0 && docID < values.length;
+ return (0xff & values[docID]);
+ }
+
+ @Override
+ protected void setInternal(int docId, long value) {
+ values[docId] = (byte) (0xFF & value);
+ }
+
+ @Override
+ protected int grow(int numDocs) {
+ values = ArrayUtil.grow(values, numDocs);
+ return values.length;
+ }
+
+ @Override
+ void read(IndexInput input, int numDocs) throws IOException {
+ if (values.length < numDocs) {
+ values = new byte[numDocs];
+ adjustSize(numDocs);
+ }
+ input.readBytes(values, 0, values.length, false);
+ maxDocID = numDocs - 1;
+ }
+
+ @Override
+ void write(IndexOutput output, int numDocs) throws IOException {
+ assert maxDocID + 1 <= numDocs;
+ output.writeBytes(values, 0, maxDocID + 1);
+ final byte zero = 0;
+ for (int i = maxDocID + 1; i < numDocs; i++) {
+ output.writeByte(zero);
+ }
+ }
+
+ @Override
+ ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input)
+ throws IOException {
+ return new FixedIntsEnumImpl(attrSource, input, type) {
+ @Override
+ protected void fillNext(LongsRef ref, IndexInput dataIn)
+ throws IOException {
+ ref.ints[ref.offset] = (0xff & dataIn.readByte());
+ }
+ };
+ }
+
+ @Override
+ void clear() {
+ super.clear();
+ values = new byte[0];
+ }
+ };
+
+ final static class ShortValues extends IndexDocValuesArray {
+ private short[] values;
+
+ ShortValues(AtomicLong bytesUsed) {
+ super(bytesUsed, RamUsageEstimator.NUM_BYTES_SHORT,
+ ValueType.FIXED_INTS_16);
+ values = new short[0];
+ }
+
+ ShortValues() {
+ super(ValueType.FIXED_INTS_16);
+ }
+
+ @Override
+ public short[] getArray() {
+ return values;
+ }
+
+ @Override
+ public long getInt(int docID) {
+ assert docID >= 0 && docID < values.length;
+ return values[docID];
+ }
+
+ @Override
+ protected void setInternal(int docId, long value) {
+ values[docId] = (short) (0xFFFF & value);
+ }
+
+ @Override
+ protected int grow(int numDocs) {
+ values = ArrayUtil.grow(values, numDocs);
+ return values.length;
+ }
+
+ @Override
+ void read(IndexInput input, int numDocs) throws IOException {
+ if (values.length < numDocs) {
+ values = new short[numDocs];
+ adjustSize(numDocs);
+ }
+ for (int i = 0; i < values.length; i++) {
+ values[i] = input.readShort();
+ }
+ maxDocID = numDocs - 1;
+ }
+
+ @Override
+ void write(IndexOutput output, int numDocs) throws IOException {
+ assert maxDocID + 1 <= numDocs;
+ for (int i = 0; i < maxDocID + 1; i++) {
+ output.writeShort(values[i]);
+ }
+ final short zero = 0;
+ for (int i = maxDocID + 1; i < numDocs; i++) {
+ output.writeShort(zero);
+ }
+ }
+
+ @Override
+ ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input)
+ throws IOException {
+ return new FixedIntsEnumImpl(attrSource, input, type) {
+ @Override
+ protected void fillNext(LongsRef ref, IndexInput dataIn)
+ throws IOException {
+ ref.ints[ref.offset] = dataIn.readShort();
+ }
+ };
+ }
+
+ @Override
+ void clear() {
+ super.clear();
+ values = new short[0];
+ }
+
+ };
+
+ final static class IntValues extends IndexDocValuesArray {
+ private int[] values;
+
+ IntValues(AtomicLong bytesUsed) {
+ super(bytesUsed, RamUsageEstimator.NUM_BYTES_INT, ValueType.FIXED_INTS_32);
+ values = new int[0];
+ }
+
+ IntValues() {
+ super(ValueType.FIXED_INTS_32);
+ }
+
+ @Override
+ public int[] getArray() {
+ return values;
+ }
+
+ @Override
+ public long getInt(int docID) {
+ assert docID >= 0 && docID < values.length;
+ return values[docID];
+ }
+
+ @Override
+ protected void setInternal(int docId, long value) {
+ values[docId] = (int) (0xFFFFFFFF & value);
+ }
+
+ @Override
+ protected int grow(int numDocs) {
+ values = ArrayUtil.grow(values, numDocs);
+ return values.length;
+ }
+
+ @Override
+ void read(IndexInput input, int numDocs) throws IOException {
+ if (values.length < numDocs) {
+ values = new int[numDocs];
+ adjustSize(numDocs);
+ }
+ for (int i = 0; i < values.length; i++) {
+ values[i] = input.readInt();
+ }
+ maxDocID = numDocs - 1;
+ }
+
+ @Override
+ void write(IndexOutput output, int numDocs) throws IOException {
+ assert maxDocID + 1 <= numDocs;
+ for (int i = 0; i < maxDocID + 1; i++) {
+ output.writeInt(values[i]);
+ }
+ for (int i = maxDocID + 1; i < numDocs; i++) {
+ output.writeInt(0);
+ }
+ }
+
+ @Override
+ ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input)
+ throws IOException {
+ return new FixedIntsEnumImpl(attrSource, input, type) {
+ @Override
+ protected void fillNext(LongsRef ref, IndexInput dataIn)
+ throws IOException {
+ ref.ints[ref.offset] = dataIn.readInt();
+ }
+ };
+ }
+
+ @Override
+ void clear() {
+ super.clear();
+ values = new int[0];
+ }
+ };
+
+ final static class LongValues extends IndexDocValuesArray {
+ private long[] values;
+
+ LongValues(AtomicLong bytesUsed) {
+ super(bytesUsed, RamUsageEstimator.NUM_BYTES_LONG,
+ ValueType.FIXED_INTS_64);
+ values = new long[0];
+ }
+
+ LongValues() {
+ super(ValueType.FIXED_INTS_64);
+ }
+
+ @Override
+ public long[] getArray() {
+ return values;
+ }
+
+ @Override
+ public long getInt(int docID) {
+ assert docID >= 0 && docID < values.length;
+ return values[docID];
+ }
+
+ @Override
+ protected void setInternal(int docId, long value) {
+ values[docId] = value;
+ }
+
+ @Override
+ protected int grow(int numDocs) {
+ values = ArrayUtil.grow(values, numDocs);
+ return values.length;
+ }
+
+ @Override
+ void read(IndexInput input, int numDocs) throws IOException {
+ if (values.length < numDocs) {
+ values = new long[numDocs];
+ adjustSize(numDocs);
+ }
+ for (int i = 0; i < values.length; i++) {
+ values[i] = input.readLong();
+ }
+ maxDocID = numDocs - 1;
+ }
+
+ @Override
+ void write(IndexOutput output, int numDocs) throws IOException {
+ assert maxDocID + 1 <= numDocs;
+ for (int i = 0; i < maxDocID + 1; i++) {
+ output.writeLong(values[i]);
+ }
+
+ for (int i = maxDocID + 1; i < numDocs; i++) {
+ output.writeLong(0l);
+ }
+ }
+
+ @Override
+ ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input)
+ throws IOException {
+ return new FixedIntsEnumImpl(attrSource, input, type) {
+ @Override
+ protected void fillNext(LongsRef ref, IndexInput dataIn)
+ throws IOException {
+ ref.ints[ref.offset] = dataIn.readLong();
+ }
+ };
+ }
+
+ @Override
+ void clear() {
+ super.clear();
+ values = new long[0];
+ }
+ };
+
+ private abstract static class FixedIntsEnumImpl extends ValuesEnum {
+ private final IndexInput dataIn;
+ private final int maxDoc;
+ private final int sizeInByte;
+ private int pos = -1;
+
+ private FixedIntsEnumImpl(AttributeSource source, IndexInput dataIn,
+ ValueType type) throws IOException {
+ super(source, type);
+ switch (type) {
+ case FIXED_INTS_16:
+ sizeInByte = 2;
+ break;
+ case FIXED_INTS_32:
+ sizeInByte = 4;
+ break;
+ case FIXED_INTS_64:
+ sizeInByte = 8;
+ break;
+ case FIXED_INTS_8:
+ sizeInByte = 1;
+ break;
+ default:
+ throw new IllegalStateException("type " + type
+ + " is not a fixed int type");
+ }
+ intsRef.offset = 0;
+ this.dataIn = dataIn;
+ maxDoc = dataIn.readInt();
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ dataIn.close();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= maxDoc) {
+ return pos = NO_MORE_DOCS;
+ }
+ assert target > pos;
+ if (target > pos + 1) {
+ dataIn
+ .seek(dataIn.getFilePointer() + ((target - pos - 1) * sizeInByte));
+ }
+ fillNext(intsRef, dataIn);
+ return pos = target;
+ }
+
+ protected abstract void fillNext(LongsRef ref, IndexInput input)
+ throws IOException;
+
+ @Override
+ public int docID() {
+ return pos;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (pos >= maxDoc) {
+ return pos = NO_MORE_DOCS;
+ }
+ return advance(pos + 1);
+ }
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/index/values/Ints.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/Ints.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/Ints.java (working copy)
@@ -33,10 +33,8 @@
private Ints() {
}
- public static Writer getWriter(Directory dir, String id,
- boolean useFixedArray, AtomicLong bytesUsed) throws IOException {
- // TODO - implement fixed?!
- return new IntsWriter(dir, id, bytesUsed);
+ public static Writer getWriter(Directory dir, String id, AtomicLong bytesUsed, ValueType type) throws IOException {
+ return new IntsWriter(dir, id, bytesUsed, type);
}
public static IndexDocValues getValues(Directory dir, String id,
Index: lucene/src/java/org/apache/lucene/index/values/IntsImpl.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/IntsImpl.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/IntsImpl.java (working copy)
@@ -18,22 +18,22 @@
*/
import java.io.IOException;
import java.util.Collection;
+import java.util.EnumMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.values.IndexDocValuesArray;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
/**
- * Stores ints packed with fixed-bit precision.
+ * Stores ints packed and fixed with fixed-bit precision.
*
* @lucene.experimental
* */
@@ -41,43 +41,56 @@
private static final String CODEC_NAME = "Ints";
private static final byte PACKED = 0x00;
- private static final byte FIXED = 0x01;
+ private static final byte FIXED_64 = 0x01;
+ private static final byte FIXED_32 = 0x02;
+ private static final byte FIXED_16 = 0x03;
+ private static final byte FIXED_8 = 0x04;
+
+ private static ValueType[] TYPES = new ValueType[] {
+ ValueType.INTS,
+ ValueType.FIXED_INTS_64,
+ ValueType.FIXED_INTS_32,
+ ValueType.FIXED_INTS_16,
+ ValueType.FIXED_INTS_8,
+
+ };
+ private static EnumMap ORDS = new EnumMap(ValueType.class);
+
+ static {
+ ORDS.put(ValueType.FIXED_INTS_16, Byte.valueOf(FIXED_16));
+ ORDS.put(ValueType.FIXED_INTS_32, Byte.valueOf(FIXED_32));
+ ORDS.put(ValueType.FIXED_INTS_64, Byte.valueOf(FIXED_64));
+ ORDS.put(ValueType.FIXED_INTS_8, Byte.valueOf(FIXED_8));
+ ORDS.put(ValueType.INTS, Byte.valueOf(PACKED));
+ }
+
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
+
+
static class IntsWriter extends Writer {
// TODO: can we bulkcopy this on a merge?
private LongsRef intsRef;
- private long[] docToValue;
+ private final IndexDocValuesArray array;
private long minValue;
private long maxValue;
private boolean started;
private final String id;
private int lastDocId = -1;
- private IndexOutput datOut;
+ private final Directory dir;
+ private final byte type;
+
- protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed)
- throws IOException {
+ protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed,
+ ValueType valueType) throws IOException {
super(bytesUsed);
- datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- DATA_EXTENSION));
- boolean success = false;
- try {
- CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
- this.id = id;
- docToValue = new long[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG); // TODO the
- // bitset
- // needs memory
- // too
- success = true;
- } finally {
- if (!success) {
- datOut.close();
- }
- }
+ this.dir = dir;
+ this.id = id;
+ array = IndexDocValuesArray.forType(valueType, bytesUsed);
+ type = ORDS.get(valueType).byteValue();
}
@Override
@@ -94,65 +107,60 @@
}
}
lastDocId = docID;
-
- if (docID >= docToValue.length) {
- final long len = docToValue.length;
- docToValue = ArrayUtil.grow(docToValue, 1 + docID);
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG
- * ((docToValue.length) - len));
- }
- docToValue[docID] = v;
+ array.set(docID, v);
}
@Override
public void finish(int docCount) throws IOException {
+ IndexOutput datOut = null;
+ boolean success = false;
try {
+ datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
+ DATA_EXTENSION));
+ CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
if (!started) {
minValue = maxValue = 0;
}
- // if we exceed the range of positive longs we must switch to fixed ints
- if ((maxValue - minValue) < (((long)1) << 63) && (maxValue - minValue) >= 0) {
- writePackedInts(docCount);
- } else {
- writeFixedInts(docCount);
+ byte headerType = type;
+ if (type == PACKED) {
+ // if we exceed the range of positive longs we must switch to fixed ints
+ if ((maxValue - minValue) < (((long)1) << 63) && (maxValue - minValue) >= 0) {
+ writePackedInts(datOut, docCount);
+ return;
+ }
+ headerType = FIXED_64;
}
-
+ datOut.writeByte(headerType);
+ datOut.writeInt(docCount);
+ array.write(datOut, docCount);
+ success = true;
} finally {
- datOut.close();
- bytesUsed
- .addAndGet(-(RamUsageEstimator.NUM_BYTES_LONG * docToValue.length));
- docToValue = null;
+ IOUtils.closeSafely(!success, datOut);
+ array.clear();
}
}
- private void writeFixedInts(int docCount) throws IOException {
- datOut.writeByte(FIXED);
- datOut.writeInt(docCount);
- for (int i = 0; i < docToValue.length; i++) {
- datOut.writeLong(docToValue[i]); // write full array - we use 0 as default
- }
- for (int i = docToValue.length; i < docCount; i++) {
- datOut.writeLong(0); // fill with defaults values
- }
- }
-
- private void writePackedInts(int docCount) throws IOException {
+ private void writePackedInts(IndexOutput datOut, int docCount) throws IOException {
datOut.writeByte(PACKED);
datOut.writeLong(minValue);
+ assert array.type() == ValueType.FIXED_INTS_64;
+ final long[] docToValue = (long[])array.getArray();
// write a default value to recognize docs without a value for that
// field
- final long defaultValue = maxValue>= 0 && minValue <=0 ? 0-minValue : ++maxValue-minValue;
+ final long defaultValue = maxValue >= 0 && minValue <= 0 ? 0 - minValue
+ : ++maxValue - minValue;
datOut.writeLong(defaultValue);
PackedInts.Writer w = PackedInts.getWriter(datOut, docCount,
- PackedInts.bitsRequired(maxValue-minValue));
- final int limit = docToValue.length > docCount ? docCount : docToValue.length;
+ PackedInts.bitsRequired(maxValue - minValue));
+ final int limit = docToValue.length > docCount ? docCount
+ : docToValue.length;
for (int i = 0; i < limit; i++) {
w.add(docToValue[i] == 0 ? defaultValue : docToValue[i] - minValue);
}
for (int i = limit; i < docCount; i++) {
w.add(defaultValue);
}
-
+
w.finish();
}
@@ -183,7 +191,7 @@
*/
static class IntsReader extends IndexDocValues {
private final IndexInput datIn;
- private final boolean packed;
+ private final byte type;
protected IntsReader(Directory dir, String id) throws IOException {
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
@@ -191,7 +199,7 @@
boolean success = false;
try {
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
- packed = PACKED == datIn.readByte();
+ type = datIn.readByte();
success = true;
} finally {
if (!success) {
@@ -206,66 +214,37 @@
*/
@Override
public Source load() throws IOException {
- final IndexInput input = (IndexInput) datIn.clone();
boolean success = false;
+ final Source source;
+ IndexDocValuesArray array = null;
+ IndexInput input = null;
try {
- final Source source = packed ? new PackedIntsSource(input)
- : new FixedIntsSource(input);
+ input = (IndexInput) datIn.clone();
+ input.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+ if (type == PACKED) {
+ source = new PackedIntsSource(input);
+ } else {
+ source = array = IndexDocValuesArray.forType(TYPES[type]);
+ }
+ if (array != null) {
+ array.read(input, input.readInt());
+ }
+
success = true;
return source;
} finally {
if (!success) {
- IOUtils.closeSafely(true, datIn);
+ IOUtils.closeSafely(true, input, datIn);
}
}
}
-
- private static class FixedIntsSource extends Source {
- private final long[] values;
- public FixedIntsSource(IndexInput dataIn) throws IOException {
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
- final int numDocs = dataIn.readInt();
- values = new long[numDocs];
- for (int i = 0; i < values.length; i++) {
- values[i] = dataIn.readLong();
- }
- }
-
- @Override
- public long getInt(int docID) {
- assert docID >= 0 && docID < values.length;
- return values[docID];
- }
-
- @Override
- public ValueType type() {
- return ValueType.INTS;
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
- return new SourceEnum(attrSource, type(), this, values.length) {
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- intsRef.ints[intsRef.offset] = values[target];
- return pos = target;
- }
- };
- }
-
- }
-
private static class PackedIntsSource extends Source {
private final long minValue;
private final long defaultValue;
private final PackedInts.Reader values;
public PackedIntsSource(IndexInput dataIn) throws IOException {
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
values = PackedInts.getReader(dataIn);
@@ -282,8 +261,7 @@
}
@Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
+ public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
return new SourceEnum(attrSource, type(), this, values.size()) {
@Override
public int advance(int target) throws IOException {
@@ -312,8 +290,13 @@
final IndexInput input = (IndexInput) datIn.clone();
boolean success = false;
try {
- ValuesEnum inst = packed ? new PackedIntsEnumImpl(source, input)
- : new FixedIntsEnumImpl(source, input);
+ input.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+ final ValuesEnum inst;
+ if (type == PACKED) {
+ inst = new PackedIntsEnumImpl(source, input);
+ } else {
+ inst = IndexDocValuesArray.forType(TYPES[type]).getDirectEnum(source, input);
+ }
success = true;
return inst;
} finally {
@@ -327,7 +310,6 @@
public ValueType type() {
return ValueType.INTS;
}
-
}
private static final class PackedIntsEnumImpl extends ValuesEnum {
@@ -343,7 +325,6 @@
super(source, ValueType.INTS);
intsRef.offset = 0;
this.dataIn = dataIn;
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
this.ints = PackedInts.getReaderIterator(dataIn);
@@ -379,51 +360,7 @@
return advance(pos + 1);
}
}
-
- private static final class FixedIntsEnumImpl extends ValuesEnum {
- private final IndexInput dataIn;
- private final int maxDoc;
- private int pos = -1;
- private FixedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
- throws IOException {
- super(source, ValueType.INTS);
- intsRef.offset = 0;
- this.dataIn = dataIn;
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
- maxDoc = dataIn.readInt();
- }
-
- @Override
- public void close() throws IOException {
- dataIn.close();
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- assert target > pos;
- if (target > pos+1) {
- dataIn.seek(dataIn.getFilePointer() + ((target - pos - 1) * 8));
- }
- intsRef.ints[intsRef.offset] = dataIn.readLong();
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
- }
- }
+
}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/index/values/ValueType.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/ValueType.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/ValueType.java (working copy)
@@ -51,7 +51,10 @@
*
*/
INTS,
-
+ FIXED_INTS_8,
+ FIXED_INTS_16,
+ FIXED_INTS_32,
+ FIXED_INTS_64,
/**
* A 32 bit floating point value. By default there is no compression
* applied. To fit custom float values into less than 32bit either a custom
Index: lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java (working copy)
@@ -71,6 +71,10 @@
case BYTES_VAR_STRAIGHT:
bytesRef = new BytesRef();
break;
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
case INTS:
intsRef = new LongsRef(1);
break;
@@ -78,6 +82,7 @@
case FLOAT_64:
floatsRef = new FloatsRef(1);
break;
+
}
}
Index: lucene/src/java/org/apache/lucene/index/values/Writer.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/values/Writer.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/index/values/Writer.java (working copy)
@@ -197,8 +197,12 @@
comp = BytesRef.getUTF8SortedAsUnicodeComparator();
}
switch (type) {
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
case INTS:
- return Ints.getWriter(directory, id, true, bytesUsed);
+ return Ints.getWriter(directory, id, bytesUsed, type);
case FLOAT_32:
return Floats.getWriter(directory, id, 4, bytesUsed);
case FLOAT_64:
@@ -221,6 +225,7 @@
case BYTES_VAR_SORTED:
return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false,
bytesUsed);
+
default:
throw new IllegalArgumentException("Unknown Values: " + type);
}
Index: lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
===================================================================
--- lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (revision 1138329)
+++ lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (working copy)
@@ -226,6 +226,9 @@
out = new DirectCFSIndexOutput(dataOut, entry, false);
} else {
entry.dir = this.directory;
+ if (directory.fileExists(name)) {
+ throw new IOException("File already exists");
+ }
out = new DirectCFSIndexOutput(directory.createOutput(name), entry,
true);
}
Index: lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java
===================================================================
--- lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (revision 1138329)
+++ lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java (working copy)
@@ -26,13 +26,17 @@
private final CompoundFileDirectory delegate;
private final String name;
- public MockCompoundFileDirectoryWrapper(String name, MockDirectoryWrapper parent, CompoundFileDirectory delegate) throws IOException {
+ public MockCompoundFileDirectoryWrapper(String name, MockDirectoryWrapper parent, CompoundFileDirectory delegate, boolean forWrite) throws IOException {
super(parent, name, 1024);
this.name = name;
this.parent = parent;
this.delegate = delegate;
- super.initForRead(Collections.emptyMap());
- parent.addFileHandle(this, name, true);
+ if (forWrite) {
+ super.initForWrite();
+ } else {
+ super.initForRead(Collections.emptyMap());
+ }
+ parent.addFileHandle(this, name, !forWrite);
}
@Override
@@ -140,4 +144,8 @@
return delegate.openInputSlice(id, offset, length, readBufferSize);
}
+ @Override
+ public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+ return delegate.createCompoundOutput(name);
+ }
}
Index: lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
===================================================================
--- lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1138329)
+++ lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (working copy)
@@ -420,9 +420,15 @@
@Override
public synchronized CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
maybeYield();
- return new MockCompoundFileDirectoryWrapper(name, this, delegate.openCompoundInput(name, bufferSize));
+ return new MockCompoundFileDirectoryWrapper(name, this, delegate.openCompoundInput(name, bufferSize), false);
}
-
+
+ @Override
+ public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+ maybeYield();
+ return new MockCompoundFileDirectoryWrapper(name, this, delegate.createCompoundOutput(name), true);
+ }
+
/** Provided for testing purposes. Use sizeInBytes() instead. */
public synchronized final long getRecomputedSizeInBytes() throws IOException {
if (!(delegate instanceof RAMDirectory))
Index: lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (revision 1138329)
+++ lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (working copy)
@@ -28,6 +28,7 @@
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store._TestHelper;
import org.apache.lucene.util._TestUtil;
Index: lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/values/TestDocValues.java (revision 1138329)
+++ lucene/src/test/org/apache/lucene/index/values/TestDocValues.java (working copy)
@@ -185,7 +185,7 @@
for (int rx = 1; rx < 63; rx++, maxV *= 2) {
Directory dir = newDirectory();
final AtomicLong trackBytes = new AtomicLong(0);
- Writer w = Ints.getWriter(dir, "test", false, trackBytes);
+ Writer w = Ints.getWriter(dir, "test", trackBytes, ValueType.INTS);
values[0] = maxMin[j];
w.add(0, values[0]);
values[1] = maxMin[j+1];
Index: lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (revision 1138329)
+++ lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (working copy)
@@ -204,7 +204,11 @@
case BYTES_VAR_STRAIGHT:
case FLOAT_32:
case FLOAT_64:
- case INTS:
+ case INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
assertEquals(msg, valuesPerIndex-1, vE_2_merged.advance(valuesPerIndex-1));
}
@@ -259,6 +263,10 @@
final int numRemainingValues = (int) (numValues - deleted.cardinality());
final int base = r.numDocs() - numRemainingValues;
switch (val) {
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
case INTS: {
IndexDocValues intsReader = getDocValues(r, val.name());
assertNotNull(intsReader);
@@ -486,7 +494,11 @@
ValueType.BYTES_VAR_SORTED, ValueType.BYTES_VAR_STRAIGHT);
private static EnumSet NUMERICS = EnumSet.of(ValueType.INTS,
- ValueType.FLOAT_32, ValueType.FLOAT_64);
+ ValueType.FIXED_INTS_16, ValueType.FIXED_INTS_32,
+ ValueType.FIXED_INTS_64,
+// ValueType.FIXED_INTS_8, // nocommit fix test enable 8
+ ValueType.FLOAT_32,
+ ValueType.FLOAT_64);
private static Index[] IDX_VALUES = new Index[] { Index.ANALYZED,
Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS,
@@ -518,6 +530,10 @@
if (isNumeric) {
switch (value) {
case INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
valField.setInt(i);
break;
case FLOAT_32:
@@ -526,6 +542,7 @@
case FLOAT_64:
valField.setFloat(2.0d * i);
break;
+
default:
fail("unexpected value " + value);
}