diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/src/java/org/apache/lucene/index/FieldInfo.java index d02b85f..7507f69 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfo.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfo.java @@ -86,7 +86,7 @@ public final class FieldInfo { public int getCodecId() { return codecId; } - + @Override public Object clone() { FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, @@ -132,6 +132,12 @@ public final class FieldInfo { } } + public void resetDocValues(ValueType v) { + if (docValues != null) { + docValues = v; + } + } + public boolean hasDocValues() { return docValues != null; } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index 094adf9..b476b56 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -139,6 +139,8 @@ final class SegmentMerger { if (fieldInfos.hasVectors()) { mergeVectors(); } + // nocommit - can we do this here after changing FI in DV merge? + fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); return mergedDocs; } @@ -252,7 +254,8 @@ final class SegmentMerger { } } final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); - fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); + //nocommit - this is currently written after merge is done since we change FI during DV merge +// fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); int docCount = 0; @@ -616,7 +619,7 @@ final class SegmentMerger { /* don't close the perDocProducers here since they are private segment producers * and will be closed once the SegmentReader goes out of scope */ } - + private MergeState mergeState; public boolean getAnyNonBulkMerges() { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java index f765654..c3ba12b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.values.IndexDocValues; +import org.apache.lucene.index.values.TypePromoter; /** * Abstract API that consumes per document values. Concrete implementations of @@ -57,6 +58,15 @@ public abstract class PerDocConsumer implements Closeable{ */ continue; } + final TypePromoter typePromoter = docValues.getTypePromoter(); + if (typePromoter == null) { + //nocommit this should happen in the segment merger? + mergeState.fieldInfo.resetDocValues(null); + continue; + } + if (mergeState.fieldInfo.getDocValues() != typePromoter.type()) { + mergeState.fieldInfo.resetDocValues(typePromoter.type()); + } final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo); assert docValuesConsumer != null; docValuesConsumer.merge(mergeState, docValues); @@ -64,4 +74,5 @@ public abstract class PerDocConsumer implements Closeable{ } } + } diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index 70f32fa..b33436a 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -566,6 +566,13 @@ public final class Bytes { } } } + + @Override + public TypePromoter getTypePromoter() { + return TypePromoter.create(type()); + } + + } static abstract class DerefBytesWriterBase extends BytesWriterBase { diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java index f6ac7ab..d7136c3 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java @@ -124,6 +124,11 @@ class FixedDerefBytesImpl { public ValueType type() { return ValueType.BYTES_FIXED_DEREF; } + + @Override + public TypePromoter getTypePromoter() { + return TypePromoter.create(type(), size); + } } } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java index b5969ae..8e35933 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java @@ -139,5 +139,10 @@ class FixedSortedBytesImpl { public ValueType type() { return ValueType.BYTES_FIXED_SORTED; } + + @Override + public TypePromoter getTypePromoter() { + return TypePromoter.create(type(), size); + } } } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java index 64357b0..5b20bc7 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java @@ -58,6 +58,7 @@ class FixedStraightBytesImpl { int version, Counter bytesUsed, IOContext context) throws IOException { super(dir, id, codecName, version, bytesUsed, context); pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)); + pool.nextBuffer(); } @Override @@ -69,7 +70,6 @@ class FixedStraightBytesImpl { throw new IllegalArgumentException("bytes arrays > " + Short.MAX_VALUE + " are not supported"); } size = bytes.length; - pool.nextBuffer(); } else if (bytes.length != size) { throw new IllegalArgumentException("expected bytes size=" + size + " but got " + bytes.length); @@ -133,11 +133,13 @@ class FixedStraightBytesImpl { @Override protected void merge(MergeState state) throws IOException { - merge = true; datOut = getOrCreateDataOut(); boolean success = false; try { - if (state.liveDocs == null && state.reader instanceof Reader ) { + if (!merge && size != -1) { + datOut.writeInt(size); + } + if (state.liveDocs == null && tryBulkMerge(state.reader)) { Reader reader = (Reader) state.reader; final int maxDocs = reader.maxDoc; if (maxDocs == 0) { @@ -168,12 +170,17 @@ class FixedStraightBytesImpl { } success = true; } finally { + merge = true; if (!success) { IOUtils.closeWhileHandlingException(datOut); } } } + protected boolean tryBulkMerge(IndexDocValues docValues) { + return docValues instanceof Reader; + } + @Override protected void mergeDoc(int docID) throws IOException { assert lastDocID < docID; @@ -181,7 +188,7 @@ class FixedStraightBytesImpl { size = bytesRef.length; datOut.writeInt(size); } - assert size == bytesRef.length; + assert size == bytesRef.length : "size: " + size + " ref: " + bytesRef.length; if (lastDocID+1 < docID) { fill(datOut, docID); } @@ -340,12 +347,15 @@ class FixedStraightBytesImpl { return new FixedStraightBytesEnum(source, cloneData(), size, maxDoc); } - - @Override public ValueType type() { return ValueType.BYTES_FIXED_STRAIGHT; } + + @Override + public TypePromoter getTypePromoter() { + return TypePromoter.create(type(), size); + } } static class FixedStraightBytesEnum extends ValuesEnum { diff --git a/lucene/src/java/org/apache/lucene/index/values/Floats.java b/lucene/src/java/org/apache/lucene/index/values/Floats.java index a3c7d57..d08e5df 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Floats.java +++ b/lucene/src/java/org/apache/lucene/index/values/Floats.java @@ -25,6 +25,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.FloatsRef; /** * Exposes {@link Writer} and reader ({@link Source}) for 32 bit and 64 bit @@ -53,31 +54,66 @@ public class Floats { } final static class FloatsWriter extends FixedStraightBytesImpl.Writer { - private final int size; + private final int size; + private final ValueType valueType; + private FloatsRef floatsRef; public FloatsWriter(Directory dir, String id, Counter bytesUsed, IOContext context, int size) throws IOException { super(dir, id, bytesUsed, context); this.bytesRef = new BytesRef(size); + assert size == 4 || size == 8; this.size = size; + valueType = size == 4 ? ValueType.FLOAT_32 : ValueType.FLOAT_64; bytesRef.length = size; } public void add(int docID, double v) throws IOException { - if (size == 8) { - bytesRef.copy(Double.doubleToRawLongBits(v)); - } else { + toBytesRef(v); + add(docID, bytesRef); + } + + private void toBytesRef(double v) { + switch(valueType) { + case FLOAT_32: bytesRef.copy(Float.floatToRawIntBits((float)v)); + break; + case FLOAT_64: + bytesRef.copy(Double.doubleToRawLongBits(v)); + break; } - add(docID, bytesRef); + } + + @Override + protected boolean tryBulkMerge(IndexDocValues docValues) { + // only bulk merge is value type is the same otherwise size differs + return super.tryBulkMerge(docValues) && docValues.type() == valueType; } @Override public void add(int docID, PerDocFieldValues docValues) throws IOException { add(docID, docValues.getFloat()); } + + @Override + protected void setNextEnum(ValuesEnum valuesEnum) { + if (valuesEnum.type() == valueType) { + super.setNextEnum(valuesEnum); + floatsRef = null; + } else { + floatsRef = valuesEnum.getFloat(); + bytesRef = new BytesRef(size); + } + } + + @Override + protected void mergeDoc(int docID) throws IOException { + if (floatsRef != null) { + toBytesRef(floatsRef.get()); + } + super.mergeDoc(docID); + } } - final static class FloatsReader extends FixedStraightBytesImpl.Reader { final IndexDocValuesArray arrayTemplate; FloatsReader(Directory dir, String id, int maxDoc, IOContext context) diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java index 305a076..354d221 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java @@ -180,6 +180,9 @@ public abstract class IndexDocValues implements Closeable { public void close() throws IOException { cache.close(this); } + + //nocommit + public abstract TypePromoter getTypePromoter(); /** * Sets the {@link SourceCache} used by this {@link IndexDocValues} instance. This diff --git a/lucene/src/java/org/apache/lucene/index/values/Ints.java b/lucene/src/java/org/apache/lucene/index/values/Ints.java index a8961a4..884f0ff 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Ints.java +++ b/lucene/src/java/org/apache/lucene/index/values/Ints.java @@ -30,6 +30,7 @@ import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LongsRef; /** * Stores ints packed and fixed with fixed-bit precision. @@ -59,6 +60,7 @@ public final class Ints { protected static final int VERSION_CURRENT = VERSION_START; private final ValueType valueType; + private LongsRef longsRef; // used for merging / type promotion public IntsWriter(Directory dir, String id, Counter bytesUsed, IOContext context, ValueType valueType) throws IOException { @@ -69,9 +71,9 @@ public final class Ints { int version, Counter bytesUsed, IOContext context, ValueType valueType) throws IOException { super(dir, id, codecName, version, bytesUsed, context); this.valueType = valueType; - final int expectedSize = getSize(valueType); - this.bytesRef = new BytesRef(expectedSize); - bytesRef.length = expectedSize; + size = getSize(valueType); + this.bytesRef = new BytesRef(size); + bytesRef.length = size; } private static int getSize(ValueType type) { @@ -91,6 +93,11 @@ public final class Ints { @Override public void add(int docID, long v) throws IOException { + toBytesRef(v); + add(docID, bytesRef); + } + + private void toBytesRef(long v) { switch (valueType) { case FIXED_INTS_64: bytesRef.copy(v); @@ -107,14 +114,39 @@ public final class Ints { default: throw new IllegalStateException("illegal type " + valueType); } - - add(docID, bytesRef); } @Override public void add(int docID, PerDocFieldValues docValues) throws IOException { add(docID, docValues.getInt()); } + + @Override + protected boolean tryBulkMerge(IndexDocValues docValues) { + // only bulk merge is value type is the same otherwise size differs + return super.tryBulkMerge(docValues) && docValues.type() == valueType; + } + + @Override + protected void setNextEnum(ValuesEnum valuesEnum) { + if (valuesEnum.type() == valueType) { + super.setNextEnum(valuesEnum); + longsRef = null; + } else { + longsRef = valuesEnum.getInt(); + bytesRef = new BytesRef(size); + bytesRef.length = size; + } + } + + @Override + protected void mergeDoc(int docID) throws IOException { + if (longsRef != null) { + // convert to bytes first + toBytesRef(longsRef.get()); + } + super.mergeDoc(docID); + } } final static class IntsReader extends FixedStraightBytesImpl.Reader { diff --git a/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java index e0b126d..5c0206d 100644 --- a/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java @@ -99,6 +99,11 @@ public class MultiIndexDocValues extends IndexDocValues { public ValueType type() { return emptySoruce.type(); } + + @Override + public TypePromoter getTypePromoter() { + return null; + } } private static class MultiValuesEnum extends ValuesEnum { @@ -266,4 +271,16 @@ public class MultiIndexDocValues extends IndexDocValues { public ValueType type() { return this.docValuesIdx[0].docValues.type(); } + + @Override + public TypePromoter getTypePromoter() { + TypePromoter promoter = docValuesIdx[0].docValues.getTypePromoter(); + for (int i = 1; i < docValuesIdx.length; i++) { + if (promoter == null) { + return null; + } + promoter = promoter.promote(docValuesIdx[i].docValues.getTypePromoter()); + } + return promoter; + } } diff --git a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java index b73d651..9a19cb1 100644 --- a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java @@ -239,6 +239,11 @@ class PackedIntValues { public ValueType type() { return ValueType.VAR_INTS; } + + @Override + public TypePromoter getTypePromoter() { + return TypePromoter.create(type()); + } } diff --git a/lucene/src/java/org/apache/lucene/index/values/TypePromoter.java b/lucene/src/java/org/apache/lucene/index/values/TypePromoter.java new file mode 100644 index 0000000..db02507 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/values/TypePromoter.java @@ -0,0 +1,127 @@ +package org.apache.lucene.index.values; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.HashMap; +import java.util.Map; + +//nocommit - javadoc +public class TypePromoter { + private static final int IS_INT = 1 << 0; + private static final int IS_BYTE = 1 << 1; + private static final int IS_FLOAT = 1 << 2; + private static final int IS_FIXED = 1 << 3 | 1 << 4; + private static final int IS_VAR = 1 << 4; + private static final int IS_STRAIGHT = 1 << 5; + private static final int IS_DEREF = 1 << 5 | 1 << 6; + private static final int IS_SORTED = 1 << 7; + private static final int IS_8_BIT = 1 << 8 | 1 << 9 | 1 << 10 | 1 << 11; + private static final int IS_16_BIT = 1 << 9 | 1 << 10 | 1 << 11; + private static final int IS_32_BIT = 1 << 10 | 1 << 11; + private static final int IS_64_BIT = 1 << 11; + protected final ValueType type; + protected final int flags; + protected int valueSize; + + final static Map FLAGS_MAP = new HashMap(); + + static { + for (ValueType type : ValueType.values()) { + TypePromoter create = create(type); + FLAGS_MAP.put(create.flags, type); + } + } + + public TypePromoter(ValueType type, int flags, int valueSize) { + super(); + this.type = type; + this.flags = flags; + this.valueSize = valueSize; + } + + public TypePromoter(ValueType type, int flags) { + this(type, flags, -1); + } + + public static TypePromoter create(ValueType type) { + return create(type, -1); + } + + public static TypePromoter create(ValueType type, int valueSize) { + if (type == null) { + return null; + } + switch (type) { + case BYTES_FIXED_DEREF: + return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_DEREF, valueSize); + case BYTES_FIXED_SORTED: + return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_SORTED, valueSize); + case BYTES_FIXED_STRAIGHT: + return new TypePromoter(type, IS_BYTE | IS_FIXED | IS_STRAIGHT, valueSize); + case BYTES_VAR_DEREF: + return new TypePromoter(type, IS_BYTE | IS_VAR | IS_DEREF); + case BYTES_VAR_SORTED: + return new TypePromoter(type, IS_BYTE | IS_VAR | IS_SORTED); + case BYTES_VAR_STRAIGHT: + return new TypePromoter(type, IS_BYTE | IS_VAR | IS_STRAIGHT); + case FIXED_INTS_16: + return new TypePromoter(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_16_BIT); + case FIXED_INTS_32: + return new TypePromoter(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_32_BIT); + case FIXED_INTS_64: + return new TypePromoter(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_64_BIT); + case FIXED_INTS_8: + return new TypePromoter(type, IS_INT | IS_FIXED | IS_STRAIGHT | IS_8_BIT); + case FLOAT_32: + return new TypePromoter(type, IS_FLOAT | IS_FIXED | IS_STRAIGHT + | IS_32_BIT); + case FLOAT_64: + return new TypePromoter(type, IS_FLOAT | IS_FIXED | IS_STRAIGHT + | IS_64_BIT); + case VAR_INTS: + return new TypePromoter(type, IS_INT | IS_VAR | IS_STRAIGHT); + default: + throw new IllegalStateException(); + } + } + + public TypePromoter promote(TypePromoter promoter) { + + int promotedFlags = promoter.flags & this.flags; + TypePromoter promoted = create(FLAGS_MAP.get(promotedFlags), valueSize); + if (promoted == null) { + return promoted; + } + if ((promoted.flags & IS_BYTE) != 0 && (promoted.flags & IS_FIXED) != 0) { + if (this.valueSize == promoter.valueSize) { + return promoted; + } + return create(FLAGS_MAP.get(promoted.flags & ~(1 << 3))); + } + return promoted; + + } + + public ValueType type() { + return type; + } + + @Override + public String toString() { + return "TypePromoter [type=" + type + ", sizeInBytes=" + valueSize + "]"; + } +} \ No newline at end of file diff --git a/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java new file mode 100644 index 0000000..c38ca75 --- /dev/null +++ b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java @@ -0,0 +1,288 @@ +package org.apache.lucene.index.values; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Random; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IndexDocValuesField; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.Before; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +public class TestTypePromotion extends LuceneTestCase { + @Before + public void setUp() throws Exception { + super.setUp(); + assumeFalse("cannot work with preflex codec", CodecProvider.getDefault() + .getDefaultFieldCodec().equals("PreFlex")); + } + + private static EnumSet INTEGERS = EnumSet.of(ValueType.VAR_INTS, + ValueType.FIXED_INTS_16, ValueType.FIXED_INTS_32, + ValueType.FIXED_INTS_64, ValueType.FIXED_INTS_8); + + private static EnumSet FLOATS = EnumSet.of(ValueType.FLOAT_32, + ValueType.FLOAT_64); + + private static EnumSet UNSORTED_BYTES = EnumSet.of( + ValueType.BYTES_FIXED_DEREF, ValueType.BYTES_FIXED_STRAIGHT, + ValueType.BYTES_VAR_STRAIGHT, ValueType.BYTES_VAR_DEREF); + + private static EnumSet SORTED_BYTES = EnumSet.of( + ValueType.BYTES_FIXED_SORTED, ValueType.BYTES_VAR_SORTED); + + public ValueType randomValueType(EnumSet typeEnum, Random random) { + ValueType[] array = typeEnum.toArray(new ValueType[0]); + return array[random.nextInt(array.length)]; + } + + private static enum TestType { + Int, Float, Byte + } + + private void runTest(EnumSet types, TestType type) + throws CorruptIndexException, IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, + writerConfig(random.nextBoolean())); + int num_1 = atLeast(200); + int num_2 = atLeast(200); + int num_3 = atLeast(200); + long[] values = new long[num_1 + num_2 + num_3]; + index(writer, new IndexDocValuesField("promote"), + randomValueType(types, random), values, 0, num_1); + writer.commit(); + + index(writer, new IndexDocValuesField("promote"), + randomValueType(types, random), values, num_1, num_2); + writer.commit(); + + if (random.nextInt(4) == 0) { + // once in a while use addIndexes + writer.optimize(); + + Directory dir_2 = newDirectory() ; + IndexWriter writer_2 = new IndexWriter(dir_2, + writerConfig(random.nextBoolean())); + index(writer_2, new IndexDocValuesField("promote"), + randomValueType(types, random), values, num_1 + num_2, num_3); + writer_2.commit(); + writer_2.close(); + if (random.nextBoolean()) { + writer.addIndexes(dir_2); + } else { + // do a real merge here + IndexReader open = IndexReader.open(dir_2); + writer.addIndexes(open); + open.close(); + } + dir_2.close(); + } else { + index(writer, new IndexDocValuesField("promote"), + randomValueType(types, random), values, num_1 + num_2, num_3); + } + + writer.optimize(); + writer.close(); + IndexReader reader = IndexReader.open(dir); + assertTrue(reader.isOptimized()); + ReaderContext topReaderContext = reader.getTopReaderContext(); + ReaderContext[] children = topReaderContext.children(); + IndexDocValues docValues = children[0].reader.docValues("promote"); + assertEquals(1, children.length); + ValuesEnum valuesEnum = docValues.getEnum(); + for (int i = 0; i < values.length; i++) { + int nextDoc = valuesEnum.nextDoc(); + assertEquals(i, nextDoc); + switch (type) { + case Byte: + assertEquals("" + i, values[i], valuesEnum.bytes().length); + break; + case Float: + assertEquals("" + i, Double.longBitsToDouble(values[i]), valuesEnum + .getFloat().get(), 0.0); + break; + case Int: + assertEquals("" + i, values[i], valuesEnum.getInt().get()); + default: + break; + } + + } + docValues.close(); + reader.close(); + dir.close(); + } + + private IndexWriterConfig writerConfig(boolean useCompoundFile) { + final IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)); + cfg.setMergePolicy(newLogMergePolicy(random)); + LogMergePolicy policy = new LogDocMergePolicy(); + cfg.setMergePolicy(policy); + policy.setUseCompoundFile(useCompoundFile); + return cfg; + } + + public void index(IndexWriter writer, IndexDocValuesField valField, + ValueType valueType, long[] values, int offset, int num) + throws CorruptIndexException, IOException { + BytesRef ref = new BytesRef(new byte[] { 1, 2, 3, 4 }); + for (int i = offset; i < offset + num; i++) { + Document doc = new Document(); + + switch (valueType) { + case VAR_INTS: + values[i] = random.nextInt(); + valField.setInt(values[i]); + break; + case FIXED_INTS_16: + values[i] = random.nextInt(Short.MAX_VALUE); + valField.setInt((short) values[i], true); + break; + case FIXED_INTS_32: + values[i] = random.nextInt(); + valField.setInt((int) values[i], true); + break; + case FIXED_INTS_64: + values[i] = random.nextLong(); + valField.setInt(values[i], true); + break; + case FLOAT_64: + double nextDouble = random.nextDouble(); + values[i] = Double.doubleToRawLongBits(nextDouble); + valField.setFloat(nextDouble); + break; + case FLOAT_32: + float nextFloat = random.nextFloat(); + values[i] = Double.doubleToRawLongBits(nextFloat); + valField.setFloat(nextFloat); + break; + case FIXED_INTS_8: + values[i] = (byte) i; + valField.setInt((byte) i, true); + break; + case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: + case BYTES_FIXED_STRAIGHT: + values[i] = ref.length = 4; + valField.setBytes(ref, valueType); + break; + case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_VAR_STRAIGHT: + values[i] = ref.length = 1 + random.nextInt(4); + valField.setBytes(ref, valueType); + break; + + default: + fail("unexpected value " + valueType); + + } + doc.add(valField); + writer.addDocument(doc); + if (random.nextInt(10) == 0) { + writer.commit(); + } + } + } + + public void testPromoteBytes() throws IOException { + runTest(UNSORTED_BYTES, TestType.Byte); + } + + public void testSortedPromoteBytes() throws IOException { + runTest(SORTED_BYTES, TestType.Byte); + } + + public void testPromotInteger() throws IOException { + runTest(INTEGERS, TestType.Int); + } + + public void testPromotFloatingPoint() throws CorruptIndexException, + IOException { + runTest(FLOATS, TestType.Float); + } + + public void testDropOnMerge() throws IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, + writerConfig(random.nextBoolean())); + int num_1 = atLeast(200); + int num_2 = atLeast(200); + int num_3 = atLeast(200); + long[] values = new long[num_1 + num_2 + num_3]; + index(writer, new IndexDocValuesField("promote"), + randomValueType(INTEGERS, random), values, 0, num_1); + writer.commit(); + + index(writer, new IndexDocValuesField("promote"), + randomValueType(UNSORTED_BYTES, random), values, num_1, num_2); + writer.commit(); + + if (random.nextInt(4) == 0) { + // once in a while use addIndexes + writer.optimize(); + + Directory dir_2 = newDirectory() ; + IndexWriter writer_2 = new IndexWriter(dir_2, + writerConfig(random.nextBoolean())); + index(writer_2, new IndexDocValuesField("promote"), + randomValueType(FLOATS, random), values, num_1 + num_2, num_3); + writer_2.commit(); + writer_2.close(); + if (random.nextBoolean()) { + writer.addIndexes(dir_2); + } else { + // do a real merge here + IndexReader open = IndexReader.open(dir_2); + writer.addIndexes(open); + open.close(); + } + dir_2.close(); + } else { + index(writer, new IndexDocValuesField("promote"), + randomValueType(SORTED_BYTES, random), values, num_1 + num_2, num_3); + writer.commit(); + } + + writer.optimize(); + writer.close(); + IndexReader reader = IndexReader.open(dir); + assertTrue(reader.isOptimized()); + ReaderContext topReaderContext = reader.getTopReaderContext(); + ReaderContext[] children = topReaderContext.children(); + IndexDocValues docValues = children[0].reader.docValues("promote"); + assertNull(docValues); // no docvalues - dropped + reader.close(); + dir.close(); + } + +}