Index: lucene/src/java/org/apache/lucene/document/DocValuesField.java =================================================================== --- lucene/src/java/org/apache/lucene/document/DocValuesField.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/document/DocValuesField.java (working copy) @@ -73,7 +73,6 @@ * * * */ -@SuppressWarnings("serial") public class DocValuesField extends AbstractField implements PerDocFieldValues { protected BytesRef bytes; Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -27,6 +27,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.values.DocValues; +import org.apache.lucene.index.values.DocValuesEnum; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -195,6 +198,9 @@ /** Status for testing of term vectors (null if term vectors could not be tested). */ public TermVectorStatus termVectorStatus; + + /** Status for testing of DocValues (null if DocValues could not be tested). */ + public DocValuesStatus docValuesStatus; } /** @@ -254,6 +260,15 @@ /** Exception thrown during term vector test (null on success) */ public Throwable error = null; } + + public static final class DocValuesStatus { + /** Number of documents tested. */ + public int docCount; + /** Total number of docValues tested. */ + public long totalValueFields; + /** Exception thrown during doc values test (null on success) */ + public Throwable error = null; + } } /** Create a new CheckIndex on the directory. */ @@ -499,6 +514,8 @@ // Test Term Vectors segInfoStat.termVectorStatus = testTermVectors(info, reader, nf); + + segInfoStat.docValuesStatus = testDocValues(info, reader); // Rethrow the first exception we encountered // This will cause stats for failed segments to be incremented properly @@ -510,6 +527,8 @@ throw new RuntimeException("Stored Field test failed"); } else if (segInfoStat.termVectorStatus.error != null) { throw new RuntimeException("Term Vector test failed"); + } else if (segInfoStat.docValuesStatus.error != null) { + throw new RuntimeException("DocValues test failed"); } msg(""); @@ -920,7 +939,61 @@ return status; } + + private Status.DocValuesStatus testDocValues(SegmentInfo info, + SegmentReader reader) { + final Status.DocValuesStatus status = new Status.DocValuesStatus(); + try { + if (infoStream != null) { + infoStream.print(" test: DocValues........"); + } + final FieldInfos fieldInfos = info.getFieldInfos(); + for (FieldInfo fieldInfo : fieldInfos) { + if (fieldInfo.hasDocValues()) { + status.totalValueFields++; + final PerDocValues perDocValues = reader.perDocValues(); + final DocValues docValues = perDocValues.docValues(fieldInfo.name); + if (docValues == null) { + continue; + } + final DocValuesEnum values = docValues.getEnum(); + while (values.nextDoc() != DocValuesEnum.NO_MORE_DOCS) { + switch (fieldInfo.docValues) { + case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: + case BYTES_FIXED_STRAIGHT: + case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_VAR_STRAIGHT: + values.bytes(); + break; + case FLOAT_32: + case FLOAT_64: + values.getFloat(); + break; + case INTS: + values.getInt(); + break; + default: + throw new IllegalArgumentException("Field: " + fieldInfo.name + + " - no such DocValues type: " + fieldInfo.docValues); + } + } + } + } + msg("OK [" + status.docCount + " total doc Count; Num DocValues Fields " + + status.totalValueFields); + } catch (Throwable e) { + msg("ERROR [" + String.valueOf(e.getMessage()) + "]"); + status.error = e; + if (infoStream != null) { + e.printStackTrace(infoStream); + } + } + return status; + } + /** * Test term vectors for a segment. */ Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision 1102677) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (working copy) @@ -251,7 +251,7 @@ fieldsWriter.addField(field, fp.fieldInfo); } if (field.hasDocValues()) { - final DocValuesConsumer docValuesConsumer = docValuesConsumer(docState, fp.fieldInfo, fieldInfos); + final DocValuesConsumer docValuesConsumer = docValuesConsumer(docState, fp.fieldInfo); docValuesConsumer.add(docState.docID, field.getDocValues()); } } @@ -292,7 +292,7 @@ final private Map docValues = new HashMap(); final private Map perDocConsumers = new HashMap(); - DocValuesConsumer docValuesConsumer(DocState docState, FieldInfo fieldInfo, FieldInfos infos) + DocValuesConsumer docValuesConsumer(DocState docState, FieldInfo fieldInfo) throws IOException { DocValuesConsumer docValuesConsumer = docValues.get(fieldInfo.name); if (docValuesConsumer != null) { @@ -303,12 +303,12 @@ PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getCodecId()); SegmentCodecs codecs = perDocWriteState.segmentCodecs; assert codecs.codecs.length > fieldInfo.getCodecId(); - Codec codec = codecs.codecs[fieldInfo.getCodecId()]; perDocConsumer = codec.docsConsumer(perDocWriteState); perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer); } docValuesConsumer = perDocConsumer.addValuesField(fieldInfo); + fieldInfo.commitDocValues(); docValues.put(fieldInfo.name, docValuesConsumer); return docValuesConsumer; } Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1102677) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -127,6 +127,7 @@ } private boolean vectorsCommitted; + private boolean docValuesCommitted; /** * Reverts all uncommitted changes on this {@link FieldInfo} @@ -138,6 +139,10 @@ storePositionWithTermVector = false; storeTermVector = false; } + + if (docValues != null && !docValuesCommitted) { + docValues = null; + } } /** @@ -150,4 +155,9 @@ assert storeTermVector; vectorsCommitted = true; } + + void commitDocValues() { + assert hasDocValues(); + docValuesCommitted = true; + } } Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1102677) +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -750,5 +750,5 @@ } return roFis; } - + } Index: lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (working copy) @@ -317,22 +317,22 @@ } private final class PerDocConsumers extends PerDocConsumer { - private final ArrayList consumers = new ArrayList(); + private final PerDocConsumer[] consumers; + private final Codec[] codecs; + private final PerDocWriteState state; public PerDocConsumers(PerDocWriteState state) throws IOException { assert segmentCodecs == state.segmentCodecs; - final Codec[] codecs = segmentCodecs.codecs; - for (int i = 0; i < codecs.length; i++) { - consumers.add(codecs[i].docsConsumer(new PerDocWriteState(state, i))); - } + this.state = state; + codecs = segmentCodecs.codecs; + consumers = new PerDocConsumer[codecs.length]; } public void close() throws IOException { - Iterator it = consumers.iterator(); IOException err = null; - while (it.hasNext()) { + for (int i = 0; i < consumers.length; i++) { try { - PerDocConsumer next = it.next(); + final PerDocConsumer next = consumers[i]; if (next != null) { next.close(); } @@ -351,10 +351,13 @@ @Override public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { - assert field.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; - final PerDocConsumer perDoc = consumers.get(field.getCodecId()); + final int codecId = field.getCodecId(); + assert codecId != FieldInfo.UNASSIGNED_CODEC_ID; + PerDocConsumer perDoc = consumers[codecId]; if (perDoc == null) { - return null; + perDoc = codecs[codecId].docsConsumer(new PerDocWriteState(state, codecId)); + assert perDoc != null; + consumers[codecId] = perDoc; } return perDoc.addValuesField(field); } Index: lucene/src/java/org/apache/lucene/index/codecs/Codec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/Codec.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/index/codecs/Codec.java (working copy) @@ -73,4 +73,5 @@ public String toString() { return name; } + } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (working copy) @@ -77,6 +77,7 @@ Writer.INDEX_EXTENSION)); assert dir.fileExists(IndexFileNames.segmentFileName(filename, "", Writer.INDEX_EXTENSION)); + // until here all types use an index case BYTES_FIXED_STRAIGHT: case FLOAT_32: case FLOAT_64: Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (revision 1102657) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (working copy) @@ -84,11 +84,11 @@ @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; + throw new UnsupportedOperationException("PerDocConsumer is not supported by Preflex codec"); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return null; + throw new UnsupportedOperationException("PerDocValues is not supported by Preflex codec"); } } Index: lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (revision 1102657) +++ lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (working copy) @@ -23,9 +23,13 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.DocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; // javadoc +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.values.Type; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.Version; import org.apache.lucene.util._TestUtil; @@ -44,6 +48,9 @@ int flushAt; private double flushAtFactor = 1.0; private boolean getReaderCalled; + private final int fixedBytesLength; + private final long docValuesFieldPrefix; + private CodecProvider codecProvider; // Randomly calls Thread.yield so we mixup thread scheduling private static final class MockIndexWriter extends IndexWriter { @@ -91,6 +98,9 @@ System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec()); w.setInfoStream(System.out); } + fixedBytesLength = 37; + docValuesFieldPrefix = r.nextLong(); + codecProvider = w.getConfig().getCodecProvider(); } /** @@ -98,10 +108,57 @@ * @see IndexWriter#addDocument(Document) */ public void addDocument(Document doc) throws IOException { + randomPerDocFieldValues(r, doc); w.addDocument(doc); + maybeCommit(); } + + private void randomPerDocFieldValues(Random random, Document doc) { + + Type[] values = Type.values(); + Type type = values[random.nextInt(values.length)]; + String name = "random_" + type.name() + "" + docValuesFieldPrefix; + if ("PreFlex".equals(codecProvider.getFieldCodec(name)) || doc.getFieldable(name) != null) + return; + DocValuesField docValuesField = new DocValuesField(name); + switch (type) { + case BYTES_FIXED_DEREF: + case BYTES_FIXED_SORTED: + case BYTES_FIXED_STRAIGHT: + final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength); + BytesRef fixedRef = new BytesRef(randomUnicodeString); + if (fixedRef.length > fixedBytesLength) { + fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength); + } else { + fixedRef.grow(fixedBytesLength); + fixedRef.length = fixedBytesLength; + } + + docValuesField.setBytes(fixedRef, type); + break; + case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_VAR_STRAIGHT: + BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200)); + docValuesField.setBytes(ref, type); + break; + case FLOAT_32: + docValuesField.setFloat(random.nextFloat()); + break; + case FLOAT_64: + docValuesField.setFloat(random.nextDouble()); + break; + case INTS: + docValuesField.setInt(random.nextInt()); + break; + default: + throw new IllegalArgumentException("no such type: " + type); + } + doc.add(docValuesField); + } + private void maybeCommit() throws IOException { if (docCount++ == flushAt) { if (LuceneTestCase.VERBOSE) {