Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (revision 1177057) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (working copy) @@ -137,22 +137,22 @@ StandardPostingsReader.files(dir, segmentInfo, codecId, files); BlockTermsReader.files(dir, segmentInfo, codecId, files); FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); } @Override public void getExtensions(Set extensions) { StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + DefaultDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (revision 1177057) +++ lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (working copy) @@ -203,13 +203,13 @@ StandardPostingsReader.files(dir, segmentInfo, id, files); BlockTermsReader.files(dir, segmentInfo, id, files); FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); } @Override public void getExtensions(Set extensions) { getStandardExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + DefaultDocValuesConsumer.getExtensions(extensions); } public static void getStandardExtensions(Set extensions) { @@ -221,12 +221,12 @@ @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (working copy) @@ -134,22 +134,22 @@ public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { StandardPostingsReader.files(dir, segmentInfo, codecID, files); BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); } @Override public void getExtensions(Set extensions) { StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + DefaultDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (revision 0) @@ -0,0 +1,143 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.values.Bytes; +import org.apache.lucene.index.values.Floats; +import org.apache.lucene.index.values.IndexDocValues; +import org.apache.lucene.index.values.Ints; +import org.apache.lucene.index.values.ValueType; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; + +/** + * Abstract base class for PerDocValues implementations + * @lucene.experimental + */ +public abstract class DocValuesReaderBase extends PerDocValues { + + protected abstract void closeInternal(Collection closeables) throws IOException; + protected abstract Map docValues(); + + @Override + public void close() throws IOException { + closeInternal(docValues().values()); + } + + @Override + public IndexDocValues docValues(String field) throws IOException { + return docValues().get(field); + } + + @Override + public Collection fields() { + return docValues().keySet(); + } + + public Comparator getComparator() throws IOException { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + // Only opens files... doesn't actually load any values + protected TreeMap load(FieldInfos fieldInfos, + String segment, int docCount, Directory dir, int codecId, IOContext context) + throws IOException { + TreeMap values = new TreeMap(); + boolean success = false; + try { + + for (FieldInfo fieldInfo : fieldInfos) { + if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { + final String field = fieldInfo.name; + // TODO can we have a compound file per segment and codec for + // docvalues? + final String id = DefaultDocValuesConsumer.docValuesId(segment, + codecId, fieldInfo.number); + values.put(field, + loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context)); + } + } + success = true; + } finally { + if (!success) { + // if we fail we must close all opened resources if there are any + closeInternal(values.values()); + } + } + return values; + } + + /** + * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}. + * Codecs that use different implementations for a certain {@link ValueType} can + * simply override this method and return their custom implementations. + * + * @param docCount + * number of documents in the segment + * @param dir + * the {@link Directory} to load the {@link IndexDocValues} from + * @param id + * the unique file ID within the segment + * @param type + * the type to load + * @return a {@link IndexDocValues} instance for the given type + * @throws IOException + * if an {@link IOException} occurs + * @throws IllegalArgumentException + * if the given {@link ValueType} is not supported + */ + protected IndexDocValues loadDocValues(int docCount, Directory dir, String id, + ValueType type, IOContext context) throws IOException { + switch (type) { + case FIXED_INTS_16: + case FIXED_INTS_32: + case FIXED_INTS_64: + case FIXED_INTS_8: + case VAR_INTS: + return Ints.getValues(dir, id, docCount, type, context); + case FLOAT_32: + return Floats.getValues(dir, id, docCount, context); + case FLOAT_64: + return Floats.getValues(dir, id, docCount, context); + case BYTES_FIXED_STRAIGHT: + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context); + case BYTES_FIXED_DEREF: + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context); + case BYTES_FIXED_SORTED: + return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context); + case BYTES_VAR_STRAIGHT: + return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context); + case BYTES_VAR_DEREF: + return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context); + case BYTES_VAR_SORTED: + return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context); + default: + throw new IllegalStateException("unrecognized index values mode " + type); + } + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (working copy) @@ -18,7 +18,6 @@ */ import java.io.IOException; -import java.util.Comparator; import java.util.Set; import org.apache.lucene.index.FieldInfo; @@ -26,113 +25,51 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.values.Writer; +import org.apache.lucene.index.codecs.DocValuesWriterBase; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Counter; /** - * + * Default PerDocConsumer implementation that uses compound file. * @lucene.experimental */ -public class DefaultDocValuesConsumer extends PerDocConsumer { - private final String segmentName; - private final int codecId; +public class DefaultDocValuesConsumer extends DocValuesWriterBase { private final Directory directory; - private final Counter bytesUsed; - private final Comparator comparator; - private boolean useCompoundFile; - private final IOContext context; - public DefaultDocValuesConsumer(PerDocWriteState state, Comparator comparator, boolean useCompoundFile) throws IOException { - this.segmentName = state.segmentName; - this.codecId = state.codecId; - this.bytesUsed = state.bytesUsed; - this.context = state.context; + public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException { + super(state); //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? - this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory, - IndexFileNames.segmentFileName(segmentName, codecId, - IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory; - this.comparator = comparator; - this.useCompoundFile = useCompoundFile; + this.directory = new CompoundFileDirectory(state.directory, + IndexFileNames.segmentFileName(state.segmentName, state.codecId, + IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true); } - - public void close() throws IOException { - if (useCompoundFile) { - this.directory.close(); - } + + @Override + protected Directory getDirectory() { + return directory; } @Override - public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { - return Writer.create(field.getDocValues(), - docValuesId(segmentName, codecId, field.number), - directory, comparator, bytesUsed, context); + public void close() throws IOException { + this.directory.close(); } - + @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, - Set files, boolean useCompoundFile) throws IOException { + public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - String filename = docValuesId(segmentInfo.name, codecId, - fieldInfo.number); - if (useCompoundFile) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - return; - } else { - switch (fieldInfo.getDocValues()) { - case BYTES_FIXED_DEREF: - case BYTES_VAR_DEREF: - case BYTES_VAR_SORTED: - case BYTES_FIXED_SORTED: - case BYTES_VAR_STRAIGHT: - files.add(IndexFileNames.segmentFileName(filename, "", - Writer.INDEX_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(filename, "", - Writer.INDEX_EXTENSION)); - // until here all types use an index - case BYTES_FIXED_STRAIGHT: - case FLOAT_32: - case FLOAT_64: - case VAR_INTS: - case FIXED_INTS_16: - case FIXED_INTS_32: - case FIXED_INTS_64: - case FIXED_INTS_8: - files.add(IndexFileNames.segmentFileName(filename, "", - Writer.DATA_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(filename, "", - Writer.DATA_EXTENSION)); - break; - - default: - assert false; - } - } + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); + return; } } } - - static String docValuesId(String segmentsName, int codecID, int fieldId) { - return segmentsName + "_" + codecID + "-" + fieldId; + public static void getExtensions(Set extensions) { + extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); + extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); } - - public static void getDocValuesExtensions(Set extensions, boolean useCompoundFile) { - if (useCompoundFile) { - extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); - extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); - } else { - extensions.add(Writer.DATA_EXTENSION); - extensions.add(Writer.INDEX_EXTENSION); - } - } - } Index: lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (revision 0) @@ -0,0 +1,68 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.values.Writer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; + +/** + * Abstract base class for PerDocConsumer implementations + * @lucene.experimental + */ +public abstract class DocValuesWriterBase extends PerDocConsumer { + private final String segmentName; + private final int codecId; + private final Counter bytesUsed; + private final IOContext context; + + protected DocValuesWriterBase(PerDocWriteState state) { + this.segmentName = state.segmentName; + this.codecId = state.codecId; + this.bytesUsed = state.bytesUsed; + this.context = state.context; + } + + protected abstract Directory getDirectory(); + + @Override + public void close() throws IOException { + } + + @Override + public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { + return Writer.create(field.getDocValues(), + docValuesId(segmentName, codecId, field.number), + getDirectory(), getComparator(), bytesUsed, context); + } + + public static String docValuesId(String segmentsName, int codecID, int fieldId) { + return segmentsName + "_" + codecID + "-" + fieldId; + } + + public Comparator getComparator() throws IOException { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (working copy) @@ -69,23 +69,23 @@ @Override public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { files.add(getPostingsFileName(segmentInfo.name, id)); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); } @Override public void getExtensions(Set extensions) { extensions.add(POSTINGS_EXTENSION); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + DefaultDocValuesConsumer.getExtensions(extensions); } // TODO: would be great if these used a plain text impl @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (working copy) @@ -110,19 +110,19 @@ public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { StandardPostingsReader.files(dir, segmentInfo, codecID, files); BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); } @Override public void getExtensions(Set extensions) { getStandardExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); } public static void getStandardExtensions(Set extensions) { extensions.add(FREQ_EXTENSION); extensions.add(PROX_EXTENSION); BlockTreeTermsReader.getExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); } @Override @@ -132,11 +132,11 @@ @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (working copy) @@ -787,22 +787,22 @@ @Override public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION)); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS()); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); } @Override public void getExtensions(Set extensions) { extensions.add(EXTENSION); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + DefaultDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new DefaultDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE); + return new DefaultDocValuesProducer(state); } } Index: lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (working copy) @@ -51,5 +51,4 @@ * Returns all fields this {@link PerDocValues} contains values for. */ public abstract Collection fields(); - } Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java (revision 0) @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.DocValuesReaderBase; +import org.apache.lucene.index.values.IndexDocValues; +import org.apache.lucene.util.IOUtils; + +/** + * Implementation of PerDocValues that uses separate files. + * @lucene.experimental + */ +public class SepDocValuesProducer extends DocValuesReaderBase { + private final TreeMap docValues; + + /** + * Creates a new {@link SepDocValuesProducer} instance and loads all + * {@link IndexDocValues} instances for this segment and codec. + */ + public SepDocValuesProducer(SegmentReadState state) throws IOException { + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context); + } + + @Override + protected Map docValues() { + return docValues; + } + + @Override + protected void closeInternal(Collection closeables) throws IOException { + IOUtils.close(closeables); + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (revision 0) @@ -0,0 +1,92 @@ +package org.apache.lucene.index.codecs.sep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.DocValuesWriterBase; +import org.apache.lucene.index.values.Writer; +import org.apache.lucene.store.Directory; + +/** + * Implementation of PerDocConsumer that uses separate files. + * @lucene.experimental + */ +public class SepDocValuesConsumer extends DocValuesWriterBase { + private final Directory directory; + + public SepDocValuesConsumer(PerDocWriteState state) throws IOException { + super(state); + this.directory = state.directory; + } + + @Override + protected Directory getDirectory() { + return directory; + } + + @SuppressWarnings("fallthrough") + public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, + Set files) throws IOException { + FieldInfos fieldInfos = segmentInfo.getFieldInfos(); + for (FieldInfo fieldInfo : fieldInfos) { + if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { + String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number); + switch (fieldInfo.getDocValues()) { + case BYTES_FIXED_DEREF: + case BYTES_VAR_DEREF: + case BYTES_VAR_SORTED: + case BYTES_FIXED_SORTED: + case BYTES_VAR_STRAIGHT: + files.add(IndexFileNames.segmentFileName(filename, "", + Writer.INDEX_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(filename, "", + Writer.INDEX_EXTENSION)); + // until here all types use an index + case BYTES_FIXED_STRAIGHT: + case FLOAT_32: + case FLOAT_64: + case VAR_INTS: + case FIXED_INTS_16: + case FIXED_INTS_32: + case FIXED_INTS_64: + case FIXED_INTS_8: + files.add(IndexFileNames.segmentFileName(filename, "", + Writer.DATA_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(filename, "", + Writer.DATA_EXTENSION)); + break; + + default: + assert false; + } + } + } + } + + public static void getExtensions(Set extensions) { + extensions.add(Writer.DATA_EXTENSION); + extensions.add(Writer.INDEX_EXTENSION); + } +} Property changes on: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/index/codecs/Codec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/Codec.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/Codec.java (working copy) @@ -18,7 +18,6 @@ */ import java.io.IOException; -import java.util.Comparator; import java.util.Set; import org.apache.lucene.index.PerDocWriteState; @@ -26,7 +25,6 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; /** @lucene.experimental */ public abstract class Codec { @@ -34,22 +32,9 @@ /** Unique name that's used to retrieve this codec when * reading the index */ public final String name; - protected final boolean dvUseCompoundFile; - protected final Comparator docValuesSortComparator; protected Codec(String name) { - this(name, true); - } - - protected Codec(String name, boolean docValuesUseCompoundFile) { - this(name, docValuesUseCompoundFile, BytesRef.getUTF8SortedAsUnicodeComparator()); - } - - protected Codec(String name, boolean docValuesUseCompoundFile, - Comparator docValuesSortComparator) { this.name = name; - this.dvUseCompoundFile = docValuesUseCompoundFile; - this.docValuesSortComparator = docValuesSortComparator; } /** Writes a new segment */ @@ -87,38 +72,6 @@ /** Records all file extensions this codec uses */ public abstract void getExtensions(Set extensions); - - /** - * Returns true iff compound file should be used for - * IndexDocValues, otherwise false. The default is - * true. - *

- * NOTE: To change the default value you need to subclass a {@link Codec} with - * a distinct name since this value is final and should not be changed to - * prevent the risk of a index corruption. This setting is private to a - * {@link Codec}. If you intend to change this value on an existing - * {@link Codec} re-indexing is required. - * - * @return true iff compound file should be used for - * IndexDocValues, otherwise false. - */ - public boolean getDocValuesUseCFS() { - return dvUseCompoundFile; - } - - /** - * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants. - * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}. - *

- * NOTE: To change the default value you need to subclass a {@link Codec} with - * a distinct name since this value is final and should not be changed to - * prevent the risk of a index corruption. This setting is private to a - * {@link Codec}. If you intend to change this value on an existing - * {@link Codec} re-indexing is required. - */ - public Comparator getDocValuesSortComparator() { - return docValuesSortComparator; - } @Override public String toString() { Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (revision 1177057) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (working copy) @@ -16,184 +16,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; +import java.util.Map; import java.util.TreeMap; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.values.Bytes; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.DocValuesReaderBase; import org.apache.lucene.index.values.IndexDocValues; -import org.apache.lucene.index.values.Floats; -import org.apache.lucene.index.values.Ints; -import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; /** - * Abstract base class for FieldsProducer implementations supporting - * {@link IndexDocValues}. - * + * Default PerDocValues implementation that uses compound file. * @lucene.experimental */ -public class DefaultDocValuesProducer extends PerDocValues { - +public class DefaultDocValuesProducer extends DocValuesReaderBase { protected final TreeMap docValues; - private final boolean useCompoundFile; - private final Closeable cfs; - private final Comparator sortComparator; + private final Directory cfs; /** - * * Creates a new {@link DefaultDocValuesProducer} instance and loads all * {@link IndexDocValues} instances for this segment and codec. - * - * @param si - * the segment info to load the {@link IndexDocValues} for. - * @param dir - * the directory to load the {@link IndexDocValues} from. - * @param fieldInfo - * the {@link FieldInfos} - * @param codecId - * the codec ID - * @param useCompoundFile - * if true this producer opens a compound file to read - * IndexDocValues fields, otherwise each field defines its own set of - * files. - * @param sortComparator - * defines the sort order for sorted IndexDocValues variants - * @throws IOException - * if an {@link IOException} occurs */ - public DefaultDocValuesProducer(SegmentInfo si, Directory dir, - FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator sortComparator, IOContext context) throws IOException { - this.useCompoundFile = useCompoundFile; - this.sortComparator = sortComparator; - final Directory directory; - if (useCompoundFile) { - cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); - } else { - cfs = null; - directory = dir; - } - docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context); + public DefaultDocValuesProducer(SegmentReadState state) throws IOException { + cfs = new CompoundFileDirectory(state.dir, + IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), + state.context, false); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context); } - - /** - * Returns a {@link IndexDocValues} instance for the given field name or - * null if this field has no {@link IndexDocValues}. - */ + @Override - public IndexDocValues docValues(String field) throws IOException { - return docValues.get(field); + protected Map docValues() { + return docValues; } - // Only opens files... doesn't actually load any values - protected TreeMap load(FieldInfos fieldInfos, - String segment, int docCount, Directory dir, int codecId, IOContext context) - throws IOException { - TreeMap values = new TreeMap(); - boolean success = false; - try { - - for (FieldInfo fieldInfo : fieldInfos) { - if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { - final String field = fieldInfo.name; - // TODO can we have a compound file per segment and codec for - // docvalues? - final String id = DefaultDocValuesConsumer.docValuesId(segment, - codecId, fieldInfo.number); - values.put(field, - loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context)); - } - } - success = true; - } finally { - if (!success) { - // if we fail we must close all opened resources if there are any - closeInternal(values.values()); - } - } - return values; - } - - - /** - * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}. - * Codecs that use different implementations for a certain {@link ValueType} can - * simply override this method and return their custom implementations. - * - * @param docCount - * number of documents in the segment - * @param dir - * the {@link Directory} to load the {@link IndexDocValues} from - * @param id - * the unique file ID within the segment - * @param type - * the type to load - * @param sortComparator byte comparator used by sorted variants - * @return a {@link IndexDocValues} instance for the given type - * @throws IOException - * if an {@link IOException} occurs - * @throws IllegalArgumentException - * if the given {@link ValueType} is not supported - */ - protected IndexDocValues loadDocValues(int docCount, Directory dir, String id, - ValueType type, Comparator sortComparator, IOContext context) throws IOException { - switch (type) { - case FIXED_INTS_16: - case FIXED_INTS_32: - case FIXED_INTS_64: - case FIXED_INTS_8: - case VAR_INTS: - return Ints.getValues(dir, id, docCount, type, context); - case FLOAT_32: - return Floats.getValues(dir, id, docCount, context); - case FLOAT_64: - return Floats.getValues(dir, id, docCount, context); - case BYTES_FIXED_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context); - case BYTES_FIXED_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context); - case BYTES_FIXED_SORTED: - return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context); - case BYTES_VAR_STRAIGHT: - return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context); - case BYTES_VAR_DEREF: - return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context); - case BYTES_VAR_SORTED: - return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context); - default: - throw new IllegalStateException("unrecognized index values mode " + type); - } - } - - public void close() throws IOException { - closeInternal(docValues.values()); - } - - private void closeInternal(Collection closeables) throws IOException { - final Collection toClose; - if (useCompoundFile) { - final ArrayList list = new ArrayList(closeables); - list.add(cfs); - toClose = list; - } else { - toClose = closeables; - } - IOUtils.close(toClose); - } - @Override - public Collection fields() { - return docValues.keySet(); + protected void closeInternal(Collection closeables) throws IOException { + final ArrayList list = new ArrayList(closeables); + list.add(cfs); + IOUtils.close(list); } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (revision 1177057) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (working copy) @@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.PostingsReaderBase; @@ -230,7 +230,7 @@ SepPostingsReader.files(segmentInfo, codecId, files); BlockTermsReader.files(dir, segmentInfo, codecId, files); FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS()); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); } @Override @@ -238,16 +238,16 @@ SepPostingsWriter.getExtensions(extensions); BlockTermsReader.getExtensions(extensions); FixedGapTermsIndexReader.getIndexExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + SepDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new SepDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new SepDocValuesProducer(state); } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (revision 1177057) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (working copy) @@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.PostingsReaderBase; @@ -207,7 +207,7 @@ SepPostingsReader.files(segmentInfo, codecId, files); BlockTermsReader.files(dir, segmentInfo, codecId, files); FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS()); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); } @Override @@ -215,16 +215,16 @@ SepPostingsWriter.getExtensions(extensions); BlockTermsReader.getExtensions(extensions); FixedGapTermsIndexReader.getIndexExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + SepDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new SepDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new SepDocValuesProducer(state); } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (revision 1177057) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (working copy) @@ -25,13 +25,11 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; @@ -40,6 +38,8 @@ import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.store.Directory; @@ -139,28 +139,28 @@ SepPostingsReader.files(segmentInfo, codecId, files); BlockTermsReader.files(dir, segmentInfo, codecId, files); FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS()); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); } @Override public void getExtensions(Set extensions) { getSepExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); } public static void getSepExtensions(Set extensions) { SepPostingsWriter.getExtensions(extensions); BlockTermsReader.getExtensions(extensions); FixedGapTermsIndexReader.getIndexExtensions(extensions); + SepDocValuesConsumer.getExtensions(extensions); } @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + return new SepDocValuesConsumer(state); } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + return new SepDocValuesProducer(state); } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (revision 1177057) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (working copy) @@ -58,6 +58,8 @@ import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; import org.apache.lucene.index.codecs.standard.StandardPostingsReader; @@ -75,17 +77,13 @@ */ public class MockRandomCodec extends Codec { - + private final boolean useSepDocValues; private final Random seedRandom; private final String SEED_EXT = "sd"; public MockRandomCodec(Random random) { - this(random, "MockRandom", true); - - } - - protected MockRandomCodec(Random random, String name, boolean docValuesUseCompoundFile) { - super(name, docValuesUseCompoundFile); + super("MockRandom"); + this.useSepDocValues = random.nextBoolean(); this.seedRandom = new Random(random.nextLong()); } @@ -428,7 +426,11 @@ BlockTreeTermsReader.files(dir, segmentInfo, codecId, files); FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS()); + if (useSepDocValues) { + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } else { + DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } // hackish! Iterator it = files.iterator(); while(it.hasNext()) { @@ -447,7 +449,11 @@ BlockTreeTermsReader.getExtensions(extensions); FixedGapTermsIndexReader.getIndexExtensions(extensions); VariableGapTermsIndexReader.getIndexExtensions(extensions); - DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS()); + if (useSepDocValues) { + SepDocValuesConsumer.getExtensions(extensions); + } else { + DefaultDocValuesConsumer.getExtensions(extensions); + } extensions.add(SEED_EXT); //System.out.println("MockRandom.getExtensions return " + extensions); } @@ -455,11 +461,19 @@ // can we make this more evil? @Override public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS()); + if (useSepDocValues) { + return new SepDocValuesConsumer(state); + } else { + return new DefaultDocValuesConsumer(state); + } } @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context); + if (useSepDocValues) { + return new SepDocValuesProducer(state); + } else { + return new DefaultDocValuesProducer(state); + } } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomDocValuesCodec.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomDocValuesCodec.java (revision 1177057) +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomDocValuesCodec.java (working copy) @@ -1,15 +0,0 @@ -package org.apache.lucene.index.codecs.mockrandom; - -import java.util.Random; - -/** - * Randomly combines terms index impl w/ postings impls. and uses non-CFS format for docvalues - */ -public class MockRandomDocValuesCodec extends MockRandomCodec { - - public MockRandomDocValuesCodec(Random random) { - super(random, "MockDocValuesCodec", false); - // uses noCFS for docvalues for test coverage - } - -} Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1177392) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -43,7 +43,6 @@ import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; import org.apache.lucene.index.codecs.mocksep.MockSepCodec; import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec; -import org.apache.lucene.index.codecs.mockrandom.MockRandomDocValuesCodec; import org.apache.lucene.index.codecs.preflex.PreFlexCodec; import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; import org.apache.lucene.index.codecs.pulsing.PulsingCodec; @@ -281,8 +280,6 @@ // baseBlockSize cannot be over 127: swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp); swapCodec(new MockRandomCodec(random), cp); - // give docvalues non-cfs testcoverage - swapCodec(new MockRandomDocValuesCodec(random), cp); return cp.lookup(codec); }