Index: lucene/contrib/CHANGES.txt
===================================================================
--- lucene/contrib/CHANGES.txt (revision 1158662)
+++ lucene/contrib/CHANGES.txt (working copy)
@@ -31,9 +31,6 @@
* LUCENE-2507: Added DirectSpellChecker, which retrieves correction candidates directly
from the term dictionary using levenshtein automata. (Robert Muir)
-
- * LUCENE-2836: Add FieldCacheRewriteMethod, which rewrites MultiTermQueries
- using the FieldCache's TermsEnum. (Robert Muir)
API Changes
Index: lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
===================================================================
--- lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (revision 1158662)
+++ lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (working copy)
@@ -45,17 +45,17 @@
setMergePolicy(mergePolicy)
);
for (int x=0; x < 100; x++) {
- Document doc = TestIndexWriterReader.createDocument(x, "index", 5);
+ Document doc = DocHelper.createDocument(x, "index", 5);
iw.addDocument(doc);
}
iw.commit();
for (int x=100; x < 150; x++) {
- Document doc = TestIndexWriterReader.createDocument(x, "index2", 5);
+ Document doc = DocHelper.createDocument(x, "index2", 5);
iw.addDocument(doc);
}
iw.commit();
for (int x=150; x < 200; x++) {
- Document doc = TestIndexWriterReader.createDocument(x, "index3", 5);
+ Document doc = DocHelper.createDocument(x, "index3", 5);
iw.addDocument(doc);
}
iw.commit();
Index: lucene/contrib/contrib-build.xml
===================================================================
--- lucene/contrib/contrib-build.xml (revision 1158662)
+++ lucene/contrib/contrib-build.xml (working copy)
@@ -41,7 +41,6 @@
-
@@ -284,4 +283,4 @@
-
\ No newline at end of file
+
Index: lucene/contrib/queries/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java
===================================================================
--- lucene/contrib/queries/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java (revision 1158642)
+++ lucene/contrib/queries/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java (working copy)
@@ -1,44 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.util.automaton.RegExp;
-
-/**
- * Tests the FieldcacheRewriteMethod with random regular expressions
- */
-public class TestFieldCacheRewriteMethod extends TestRegexpRandom2 {
-
- /** Test fieldcache rewrite against filter rewrite */
- @Override
- protected void assertSame(String regexp) throws IOException {
- RegexpQuery fieldCache = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
- fieldCache.setRewriteMethod(new FieldCacheRewriteMethod());
-
- RegexpQuery filter = new RegexpQuery(new Term("field", regexp), RegExp.NONE);
- filter.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
-
- TopDocs fieldCacheDocs = searcher1.search(fieldCache, 25);
- TopDocs filterDocs = searcher2.search(filter, 25);
-
- CheckHits.checkEqual(fieldCache, fieldCacheDocs.scoreDocs, filterDocs.scoreDocs);
- }
-}
Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java
===================================================================
--- lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1158642)
+++ lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy)
@@ -1,164 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Comparator;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.OpenBitSet;
-
-/**
- * Rewrites MultiTermQueries into a filter, using the FieldCache for term enumeration.
- *
- * WARNING: This is only appropriate for single-valued unanalyzed fields. Additionally, for
- * most queries this method is actually SLOWER than using the default CONSTANT_SCORE_AUTO
- * in MultiTermQuery. This method is only faster than other methods for certain queries,
- * such as ones that enumerate many terms.
- *
- * @lucene.experimental
- */
-public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod {
-
- @Override
- public Query rewrite(IndexReader reader, MultiTermQuery query) {
- Query result = new ConstantScoreQuery(new MultiTermQueryFieldCacheWrapperFilter(query));
- result.setBoost(query.getBoost());
- return result;
- }
-
- static class MultiTermQueryFieldCacheWrapperFilter extends Filter {
-
- protected final MultiTermQuery query;
-
- /**
- * Wrap a {@link MultiTermQuery} as a Filter.
- */
- protected MultiTermQueryFieldCacheWrapperFilter(MultiTermQuery query) {
- this.query = query;
- }
-
- @Override
- public String toString() {
- // query.toString should be ok for the filter, too, if the query boost is 1.0f
- return query.toString();
- }
-
- @Override
- public final boolean equals(final Object o) {
- if (o==this) return true;
- if (o==null) return false;
- if (this.getClass().equals(o.getClass())) {
- return this.query.equals( ((MultiTermQueryFieldCacheWrapperFilter)o).query );
- }
- return false;
- }
-
- @Override
- public final int hashCode() {
- return query.hashCode();
- }
-
- /** Returns the field name for this query */
- public final String getField() { return query.getField(); }
-
- /**
- * Expert: Return the number of unique terms visited during execution of the filter.
- * If there are many of them, you may consider using another filter type
- * or optimize your total term count in index.
- *
This method is not thread safe, be sure to only call it when no filter is running!
- * If you re-use the same filter instance for another
- * search, be sure to first reset the term counter
- * with {@link #clearTotalNumberOfTerms}.
- * @see #clearTotalNumberOfTerms
- */
- public int getTotalNumberOfTerms() {
- return query.getTotalNumberOfTerms();
- }
-
- /**
- * Expert: Resets the counting of unique terms.
- * Do this before executing the filter.
- * @see #getTotalNumberOfTerms
- */
- public void clearTotalNumberOfTerms() {
- query.clearTotalNumberOfTerms();
- }
-
- /**
- * Returns a DocIdSet with documents that should be permitted in search
- * results.
- */
- @Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
- final FieldCache.DocTermsIndex fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader, query.field);
- // Cannot use FixedBitSet because we require long index (ord):
- final OpenBitSet termSet = new OpenBitSet(fcsi.numOrd());
- TermsEnum termsEnum = query.getTermsEnum(new Terms() {
-
- @Override
- public Comparator getComparator() throws IOException {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public TermsEnum iterator() throws IOException {
- return fcsi.getTermsEnum();
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return -1;
- }
-
- @Override
- public long getSumDocFreq() throws IOException {
- return -1;
- }
- });
-
- assert termsEnum != null;
- if (termsEnum.next() != null) {
- // fill into a OpenBitSet
- int termCount = 0;
- do {
- long ord = termsEnum.ord();
- if (ord > 0) {
- termSet.set(ord);
- termCount++;
- }
- } while (termsEnum.next() != null);
-
- query.incTotalNumberOfTerms(termCount);
- } else {
- return DocIdSet.EMPTY_DOCIDSET;
- }
-
- return new FieldCacheRangeFilter.FieldCacheDocIdSet(context.reader, true) {
- @Override
- boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException {
- return termSet.get(fcsi.getOrd(doc));
- }
- };
- }
- }
-}
Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1158662)
+++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy)
@@ -74,7 +74,7 @@
for (int i = 0; i < 97 ; i++) {
IndexReader reader = writer.getReader();
if (i == 0) {
- writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5)));
} else {
int previous = random.nextInt(i);
// a check if the reader is current here could fail since there might be
@@ -83,10 +83,10 @@
case 0:
case 1:
case 2:
- writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5)));
break;
case 3:
- writer.updateDocument(new Term("id", "" + previous), createDocument(
+ writer.updateDocument(new Term("id", "" + previous), DocHelper.createDocument(
previous, "x", 1 + random.nextInt(5)));
break;
case 4:
@@ -105,7 +105,7 @@
iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
writer = new IndexWriter(dir1, iwc);
assertTrue(reader.isCurrent());
- writer.addDocument(createDocument(1, "x", 1+random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(1, "x", 1+random.nextInt(5)));
assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
writer.close();
assertFalse(reader.isCurrent()); // segments written
@@ -422,7 +422,7 @@
addDir = newDirectory();
IndexWriter writer = new IndexWriter(addDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
for (int i = 0; i < NUM_INIT_DOCS; i++) {
- Document doc = createDocument(i, "addindex", 4);
+ Document doc = DocHelper.createDocument(i, "addindex", 4);
writer.addDocument(doc);
}
@@ -543,7 +543,7 @@
assertEquals(r2.maxDoc(), 100);
// add 100 documents
for (int x = 10000; x < 10000 + 100; x++) {
- Document d = createDocument(x, "index1", 5);
+ Document d = DocHelper.createDocument(x, "index1", 5);
writer.addDocument(d);
}
writer.flush(false, true);
@@ -575,25 +575,7 @@
dir1.close();
}
-
-
- public static Document createDocument(int n, String indexName, int numFields) {
- StringBuilder sb = new StringBuilder();
- Document doc = new Document();
- doc.add(new Field("id", Integer.toString(n), Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(new Field("indexname", indexName, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- sb.append("a");
- sb.append(n);
- doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- sb.append(" b");
- sb.append(n);
- for (int i = 1; i < numFields; i++) {
- doc.add(new Field("field" + (i + 1), sb.toString(), Store.YES,
- Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- }
- return doc;
- }
-
+
/*
* Delete a document by term and return the doc id
*
@@ -609,7 +591,7 @@
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMergePolicy(new LogDocMergePolicy()));
for (int i = 0; i < 100; i++) {
- w.addDocument(createDocument(i, indexName, 4));
+ w.addDocument(DocHelper.createDocument(i, indexName, 4));
if (multiSegment && (i % 10) == 0) {
}
}
@@ -622,7 +604,7 @@
public static void createIndexNoClose(boolean multiSegment, String indexName,
IndexWriter w) throws IOException {
for (int i = 0; i < 100; i++) {
- w.addDocument(createDocument(i, indexName, 4));
+ w.addDocument(DocHelper.createDocument(i, indexName, 4));
}
if (!multiSegment) {
w.optimize();
@@ -662,14 +644,14 @@
int num = atLeast(100);
for (int i = 0; i < num; i++) {
- writer.addDocument(createDocument(i, "test", 4));
+ writer.addDocument(DocHelper.createDocument(i, "test", 4));
}
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
assertTrue(warmer.warmCount > 0);
final int count = warmer.warmCount;
- writer.addDocument(createDocument(17, "test", 4));
+ writer.addDocument(DocHelper.createDocument(17, "test", 4));
writer.optimize();
assertTrue(warmer.warmCount > count);
@@ -695,7 +677,7 @@
assertEquals(100, r1.numDocs());
for (int i = 0; i < 10; i++) {
- writer.addDocument(createDocument(i, "test", 4));
+ writer.addDocument(DocHelper.createDocument(i, "test", 4));
}
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
@@ -857,7 +839,7 @@
do {
try {
for(int docUpto=0;docUpto<10;docUpto++) {
- writer.addDocument(createDocument(10*count+docUpto, "test", 4));
+ writer.addDocument(DocHelper.createDocument(10*count+docUpto, "test", 4));
}
count++;
final int limit = count*10;
Index: lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (revision 1158662)
+++ lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (working copy)
@@ -91,7 +91,7 @@
//int n = random.nextInt(2);
if (type == 0) {
int i = seq.addAndGet(1);
- Document doc = TestIndexWriterReader.createDocument(i, "index1", 10);
+ Document doc = DocHelper.createDocument(i, "index1", 10);
writer.addDocument(doc);
addCount++;
} else if (type == 1) {
Index: lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (revision 1158662)
+++ lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (working copy)
@@ -46,14 +46,14 @@
IndexWriter writer = new IndexWriter(dir, iwc);
writer.setInfoStream(VERBOSE ? System.out : null);
for (int x = 0; x < 5; x++) {
- writer.addDocument(TestIndexWriterReader.createDocument(x, "1", 2));
+ writer.addDocument(DocHelper.createDocument(x, "1", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
//System.out.println("commit1");
writer.commit();
assertEquals(1, writer.segmentInfos.size());
for (int x = 5; x < 10; x++) {
- writer.addDocument(TestIndexWriterReader.createDocument(x, "2", 2));
+ writer.addDocument(DocHelper.createDocument(x, "2", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
//System.out.println("commit2");
@@ -61,7 +61,7 @@
assertEquals(2, writer.segmentInfos.size());
for (int x = 10; x < 15; x++) {
- writer.addDocument(TestIndexWriterReader.createDocument(x, "3", 2));
+ writer.addDocument(DocHelper.createDocument(x, "3", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
@@ -174,12 +174,12 @@
**/
void part2(IndexWriter writer, RangeMergePolicy fsmp) throws Exception {
for (int x = 20; x < 25; x++) {
- writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
+ writer.addDocument(DocHelper.createDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
writer.flush(false, false);
for (int x = 25; x < 30; x++) {
- writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
+ writer.addDocument(DocHelper.createDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
writer.flush(false, false);
Index: lucene/src/test/org/apache/lucene/util/europarl.lines.txt.gz
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
===================================================================
--- lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (revision 1158662)
+++ lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (working copy)
@@ -29,6 +29,9 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -251,4 +254,21 @@
public static int numFields(Document doc) {
return doc.getFields().size();
}
+
+ public static Document createDocument(int n, String indexName, int numFields) {
+ StringBuilder sb = new StringBuilder();
+ Document doc = new Document();
+ doc.add(new Field("id", Integer.toString(n), Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+ doc.add(new Field("indexname", indexName, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+ sb.append("a");
+ sb.append(n);
+ doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+ sb.append(" b");
+ sb.append(n);
+ for (int i = 1; i < numFields; i++) {
+ doc.add(new Field("field" + (i + 1), sb.toString(), Store.YES,
+ Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+ }
+ return doc;
+ }
}