Index: lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (revision 1293829) +++ lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (working copy) @@ -18,6 +18,8 @@ */ import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -29,7 +31,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -97,10 +98,9 @@ createRandomIndex(numdocs, leftWriter, seed); createRandomIndex(numdocs, rightWriter, seed); - // TODO: maybe we should do this wrapping in another test? - leftReader = maybeWrap(leftWriter.getReader()); + leftReader = maybeWrapReader(leftWriter.getReader()); leftWriter.close(); - rightReader = maybeWrap(rightWriter.getReader()); + rightReader = maybeWrapReader(rightWriter.getReader()); rightWriter.close(); info = "left: " + leftCodec.toString() + " / right: " + rightCodec.toString(); @@ -116,12 +116,6 @@ super.tearDown(); } - static IndexReader maybeWrap(IndexReader other) throws IOException { - // TODO: bogus how we do this - IndexSearcher is = newSearcher(other); - return is.getIndexReader(); - } - /** * populates a writer with random stuff. this must be fully reproducable with the seed! */ @@ -488,8 +482,18 @@ Document rightDoc = rightReader.document(i); // TODO: I think this is bogus because we don't document what the order should be - // from these iterators, etc. I think the codec should be free to order this stuff + // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff // in whatever way it wants (e.g. maybe it packs related fields together or something) + // To fix this, we sort the fields in both documents by name, but + // we still assume that all instances with same name are in order: + Comparator comp = new Comparator() { + @Override + public int compare(IndexableField arg0, IndexableField arg1) { + return arg0.name().compareTo(arg1.name()); + } + }; + Collections.sort(leftDoc.getFields(), comp); + Collections.sort(rightDoc.getFields(), comp); Iterator leftIterator = leftDoc.iterator(); Iterator rightIterator = rightDoc.iterator(); Index: lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java (revision 1293829) +++ lucene/core/src/test/org/apache/lucene/index/TestTypePromotion.java (working copy) @@ -97,9 +97,8 @@ writer.addIndexes(dir_2); } else { // do a real merge here - IndexReader open = IndexReader.open(dir_2); - // TODO: wrap in a better way - writer.addIndexes(newSearcher(open).getIndexReader()); + IndexReader open = maybeWrapReader(IndexReader.open(dir_2)); + writer.addIndexes(open); open.close(); } dir_2.close(); Index: lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (revision 0) +++ lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (working copy) @@ -0,0 +1,169 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FilterAtomicReader; + +public final class FieldFilterAtomicReader extends FilterAtomicReader { + + private final Set fields; + private final boolean negate; + private final FieldInfos fieldInfos; + + public FieldFilterAtomicReader(AtomicReader in, Set fields, boolean negate) { + super(in); + this.fields = fields; + this.negate = negate; + this.fieldInfos = new FieldInfos(); + for (FieldInfo fi : in.getFieldInfos()) { + if (hasField(fi.name)) { + fieldInfos.add(fi); + } + } + } + + boolean hasField(String field) { + return negate ^ fields.contains(field); + } + + @Override + public FieldInfos getFieldInfos() { + return fieldInfos; + } + + @Override + public Fields getTermVectors(int docID) throws IOException { + Fields f = super.getTermVectors(docID); + if (f == null) { + return null; + } + f = new FieldFilterFields(f); + // we need to check for emptyness, so we can return null: + return (f.iterator().next() == null) ? null : f; + } + + @Override + public void document(final int docID, final StoredFieldVisitor visitor) throws CorruptIndexException, IOException { + super.document(docID, new StoredFieldVisitor() { + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException { + visitor.binaryField(fieldInfo, value, offset, length); + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + visitor.stringField(fieldInfo, value); + } + + @Override + public void intField(FieldInfo fieldInfo, int value) throws IOException { + visitor.intField(fieldInfo, value); + } + + @Override + public void longField(FieldInfo fieldInfo, long value) throws IOException { + visitor.longField(fieldInfo, value); + } + + @Override + public void floatField(FieldInfo fieldInfo, float value) throws IOException { + visitor.floatField(fieldInfo, value); + } + + @Override + public void doubleField(FieldInfo fieldInfo, double value) throws IOException { + visitor.doubleField(fieldInfo, value); + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + return hasField(fieldInfo.name) ? visitor.needsField(fieldInfo) : Status.NO; + } + }); + } + + @Override + public boolean hasNorms(String field) throws IOException { + return hasField(field) ? super.hasNorms(field) : false; + } + + @Override + public Fields fields() throws IOException { + final Fields f = super.fields(); + return (f == null) ? null : new FieldFilterFields(f); + } + + @Override + public DocValues docValues(String field) throws IOException { + return hasField(field) ? super.docValues(field) : null; + } + + @Override + public DocValues normValues(String field) throws IOException { + return hasField(field) ? super.normValues(field) : null; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("FieldFilterAtomicReader(reader="); + sb.append(in).append(", fields="); + if (negate) sb.append('!'); + return sb.append(fields).append(')').toString(); + } + + private class FieldFilterFields extends FilterFields { + public FieldFilterFields(Fields in) { + super(in); + } + + @Override + public int getUniqueFieldCount() throws IOException { + // TODO: add faster implementation! + int c = 0; + final FieldsEnum it = iterator(); + while (it.next() != null) { + c++; + } + return c; + } + + @Override + public FieldsEnum iterator() throws IOException { + return new FilterFieldsEnum(super.iterator()) { + @Override + public String next() throws IOException { + String f; + while ((f = super.next()) != null) { + if (hasField(f)) return f; + } + return null; + } + }; + } + + @Override + public Terms terms(String field) throws IOException { + return hasField(field) ? super.terms(field) : null; + } + + } + +} Index: lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (revision 0) +++ lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (working copy) Property changes on: lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +Date Author Id Revision HeadURL \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java (revision 1293829) +++ lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java (working copy) @@ -167,10 +167,7 @@ 0 < edge ? r : emptyReaders[0]) }; - // TODO: fix me, - // wrapping causes insanity when we have an already-atomic reader?! - // IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers)); - IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers), false); + IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers)); out.setSimilarity(s.getSimilarity()); return out; } Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1293829) +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -55,8 +55,8 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.CompositeReader; -import org.apache.lucene.index.FilterAtomicReader; -import org.apache.lucene.index.Fields; +import org.apache.lucene.index.FieldFilterAtomicReader; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -1349,45 +1349,56 @@ /** Sometimes wrap the IndexReader as slow, parallel or filter reader (or combinations of that) */ public static IndexReader maybeWrapReader(IndexReader r) throws IOException { - // TODO: remove this, and fix those tests to wrap before putting slow around: - final boolean wasOriginallyAtomic = r instanceof AtomicReader; if (rarely()) { + // TODO: remove this, and fix those tests to wrap before putting slow around: + final boolean wasOriginallyAtomic = r instanceof AtomicReader; for (int i = 0, c = random.nextInt(6)+1; i < c; i++) { switch(random.nextInt(4)) { case 0: r = SlowCompositeReaderWrapper.wrap(r); break; case 1: + // will create no FC insanity as Parallel*Reader has own cache key: r = (r instanceof AtomicReader) ? new ParallelAtomicReader((AtomicReader) r) : new ParallelCompositeReader((CompositeReader) r); break; case 2: - if (!wasOriginallyAtomic) { // dont wrap originally atomic readers to be composite (some tests don't like) - r = new MultiReader(r); - } + // Häckidy-Hick-Hack: this will create FC insanity, so we patch MultiReader to + // return a fake cache key, so insanity checker cannot walk along our reader: + r = new MultiReader(r) { + private final Object cacheKey = new Object(); + @Override public Object getCoreCacheKey() { return cacheKey; } + @Override public Object getCombinedCoreAndDeletesKey() { return cacheKey; } + @Override public String toString() { return "MultiReader(" + subReaders[0] + ")"; } + }; break; case 3: - if (r instanceof AtomicReader) { - r = new FilterAtomicReader((AtomicReader) r) { - @Override - public Fields fields() throws IOException { - Fields f = super.fields(); - if (f == null) { - return null; - } else { - return new FilterFields(f); - } - } - }; + final AtomicReader ar = SlowCompositeReaderWrapper.wrap(r); + final List allFields = new ArrayList(); + for (FieldInfo fi : ar.getFieldInfos()) { + allFields.add(fi.name); } + Collections.shuffle(allFields, random); + final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size()); + final Set fields = new HashSet(allFields.subList(0, end)); + // will create no FC insanity as ParallelAtomicReader has own cache key: + r = new ParallelAtomicReader( + new FieldFilterAtomicReader(ar, fields, false), + new FieldFilterAtomicReader(ar, fields, true) + ); break; default: fail("should not get here"); } } + if (wasOriginallyAtomic) { + r = SlowCompositeReaderWrapper.wrap(r); + } + if (VERBOSE) { + System.out.println("maybeWrapReader wrapped: " +r); + } } - //System.out.println(r); return r; }