Index: src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- src/java/org/apache/lucene/search/ConjunctionScorer.java (revision 751749) +++ src/java/org/apache/lucene/search/ConjunctionScorer.java (working copy) @@ -19,12 +19,14 @@ import java.io.IOException; import java.util.Collection; +import java.util.Iterator; import java.util.Arrays; import java.util.Comparator; /** Scorer for conjunctions, sets of queries, all of which are required. */ class ConjunctionScorer extends Scorer { - private final Scorer[] scorers; + private Scorer[] scorers; + private DocIdSetIterator[] disis; // also includes all scorers. private boolean firstTime=true; private boolean more; @@ -35,30 +37,47 @@ this(similarity, (Scorer[])scorers.toArray(new Scorer[scorers.size()])); } + public ConjunctionScorer(Similarity similarity, Collection scorers, Collection disis) throws IOException { + this(similarity, (Scorer[])scorers.toArray(new Scorer[scorers.size()]), disis); + } + public ConjunctionScorer(Similarity similarity, Scorer[] scorers) throws IOException { super(similarity); this.scorers = scorers; + this.disis = scorers; coord = getSimilarity().coord(this.scorers.length, this.scorers.length); } + ConjunctionScorer(Similarity similarity, Scorer[] scorers, Collection disis) throws IOException { + this(similarity, scorers); + if (disis.size() > 0) { + this.disis = new DocIdSetIterator[scorers.length + disis.size()]; + System.arraycopy(this.scorers, 0, this.disis, 0, this.scorers.length); + Iterator disisIt = disis.iterator(); + for (int i = 0; disisIt.hasNext(); i++) { + this.disis[i + this.scorers.length] = (DocIdSetIterator) disisIt.next(); + } + } + } + public int doc() { return lastDoc; } public boolean next() throws IOException { if (firstTime) return init(0); else if (more) - more = scorers[(scorers.length-1)].next(); + more = disis[(disis.length-1)].next(); return doNext(); } private boolean doNext() throws IOException { int first=0; - Scorer lastScorer = scorers[scorers.length-1]; - Scorer firstScorer; - while (more && (firstScorer=scorers[first]).doc() < (lastDoc=lastScorer.doc())) { - more = firstScorer.skipTo(lastDoc); - lastScorer = firstScorer; - first = (first == (scorers.length-1)) ? 0 : first+1; + DocIdSetIterator lastDisi = disis[disis.length-1]; + DocIdSetIterator firstDisi; + while (more && (firstDisi=disis[first]).doc() < (lastDoc=lastDisi.doc())) { + more = firstDisi.skipTo(lastDoc); + lastDisi = firstDisi; + first = (first == (disis.length-1)) ? 0 : first+1; } return more; } @@ -67,7 +86,7 @@ if (firstTime) return init(target); else if (more) - more = scorers[(scorers.length-1)].skipTo(target); + more = disis[(disis.length-1)].skipTo(target); return doNext(); } @@ -75,9 +94,9 @@ // thus skipping a check for firstTime per call to next() and skipTo() private boolean init(int target) throws IOException { firstTime=false; - more = scorers.length>1; - for (int i=0; i1; + for (int i=0; i>1); i++) { - Scorer tmp = scorers[i]; - scorers[i] = scorers[end-i-1]; - scorers[end-i-1] = tmp; + DocIdSetIterator tmp = disis[i]; + disis[i] = disis[end-i-1]; + disis[end-i-1] = tmp; } return more; @@ -113,7 +132,7 @@ public float score() throws IOException { float sum = 0.0f; - for (int i = 0; i < scorers.length; i++) { + for (int i = 0; i < scorers.length; i++) { // use scorers here, not disis. sum += scorers[i].score(); } return sum * coord; Index: src/java/org/apache/lucene/search/DisjunctionDISI.java =================================================================== --- src/java/org/apache/lucene/search/DisjunctionDISI.java (revision 0) +++ src/java/org/apache/lucene/search/DisjunctionDISI.java (revision 0) @@ -0,0 +1,195 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.List; +import java.util.Iterator; +import java.io.IOException; + +import org.apache.lucene.util.DisiDocQueue; + +/* Derived from org.apache.lucene.search.DisjunctionSumScorer of 29 Jan 2009 */ + +/** A DocIdSetIterator iterating over all docs of a given List of DocIdSetIterators. + */ +class DisjunctionDISI extends DocIdSetIterator { + /** The number of sub DocIdSetIterators. */ + private final int nrDisis; + + /** The subscorers. */ + protected final List subDisis; + + /** The minimum number of DocIdSetIterators that should match. */ + private final int minimumNrMatchers; + + /** The disiDocQueue contains all sub DocIdSetIterators ordered by their current doc(), + * with the minimum at the top. + *
The disiDocQueue is initialized the first time next() or skipTo() is called. + *
An exhausted DocIdSetIterator is immediately removed from the disiDocQueue. + *
If less than the minimumNrMatchers DocIdSetIterators + * remain in the disiDocQueue next() and skipTo() return false. + *

+ * After each to call to next() or skipTo() + * nrMatchers is the number of matching DocIdSetIterators, + * and all DocIdSetIterators are after the matching doc, or are exhausted. + */ + private DisiDocQueue disiDocQueue = null; + private int queueSize = -1; // used to avoid size() method calls on disiDocQueue + + /** The document number of the current match. */ + private int currentDoc = -1; + + /** The number of subscorers that provide the current match. */ + protected int nrMatchers = -1; + + private float currentScore = Float.NaN; + + /** Construct a DisjunctionDISI. + * @param subDisis A collection of at least two DocIdSetIterators. + * @param minimumNrMatchers The positive minimum number of DocIdSetIterators that should + * match for a document to be contained in this DisjunctionDISI. + *
When minimumNrMatchers is bigger than + * the number of subDisis, + * no matches will be produced. + *
When minimumNrMatchers equals the number of subDisis, + * it more efficient to use ConjunctionScorer. + */ + public DisjunctionDISI( List subDisis, int minimumNrMatchers) { + nrDisis = subDisis.size(); + + if (minimumNrMatchers <= 0) { + throw new IllegalArgumentException("Minimum nr of matchers must be positive"); + } + if (nrDisis <= 1) { + throw new IllegalArgumentException("There must be at least 2 DocIdSetIterators"); + } + + this.minimumNrMatchers = minimumNrMatchers; + this.subDisis = subDisis; + } + + /** Construct a DisjunctionScorer, using one as the minimum number + * of matching subscorers. + */ + public DisjunctionDISI(List subDisis) { + this(subDisis, 1); + } + + /** Called the first time next() or skipTo() is called to + * initialize disiDocQueue. + */ + private void initDisiDocQueue() throws IOException { + Iterator si = subDisis.iterator(); + disiDocQueue = new DisiDocQueue(nrDisis); + queueSize = 0; + while (si.hasNext()) { + DocIdSetIterator se = (DocIdSetIterator) si.next(); + if (se.next()) { // doc() method will be used in scorerDocQueue. + if (disiDocQueue.insert(se)) { + queueSize++; + } + } + } + } + + public boolean next() throws IOException { + if (disiDocQueue == null) { + initDisiDocQueue(); + } + return (disiDocQueue.size() >= minimumNrMatchers) + && advanceAfterCurrent(); + } + + /** Advance all DocIdSetIterators after the current document determined by the + * top of the disiDocQueue. + * Repeat until at least the minimum number of DocIdSetIterators match on the same + * document and all DocIdSetIterators are after that document or are exhausted. + *
On entry the disiDocQueue has at least minimumNrMatchers + * available. At least the DocIdSetIterator with the minimum document number will be advanced. + * @return true iff there is a match. + *
In case there is a match, currentDoc, + * and nrMatchers describe the match. + * + * @todo Investigate whether it is possible to use skipTo() when + * the minimum number of matchers is bigger than one, ie. try and use the + * character of ConjunctionScorer for the minimum number of matchers. + * Also delay calling score() on the sub scorers until the minimum number of + * matchers is reached. + *
For this, a Scorer array with minimumNrMatchers elements might + * hold Scorers at currentDoc that are temporarily popped from scorerQueue. + */ + protected boolean advanceAfterCurrent() throws IOException { + do { // repeat until minimum nr of matchers + currentDoc = disiDocQueue.topDoc(); + nrMatchers = 1; + do { // Until all subscorers are after currentDoc + if (! disiDocQueue.topNextAndAdjustElsePop()) { + if (--queueSize == 0) { + break; // nothing more to advance, check for last match. + } + } + if (disiDocQueue.topDoc() != currentDoc) { + break; // All remaining subscorers are after currentDoc. + } + nrMatchers++; + } while (true); + + if (nrMatchers >= minimumNrMatchers) { + return true; + } else if (queueSize < minimumNrMatchers) { + return false; + } + } while (true); + } + + public int doc() { return currentDoc; } + + /** Returns the number of subscorers matching the current document. + * Initially invalid, until {@link #next()} is called the first time. + */ + public int nrMatchers() { + return nrMatchers; + } + + /** Skips to the first match beyond the current whose document number is + * greater than or equal to a given target. + *
When this method is used the {@link #explain(int)} method should not be used. + * @param target The target document number. + * @return true iff there is such a match. + */ + public boolean skipTo(int target) throws IOException { + if (disiDocQueue == null) { + initDisiDocQueue(); + } + if (queueSize < minimumNrMatchers) { + return false; + } + if (target <= currentDoc) { + return true; + } + do { + if (disiDocQueue.topDoc() >= target) { + return advanceAfterCurrent(); + } else if (! disiDocQueue.topSkipToAndAdjustElsePop(target)) { + if (--queueSize < minimumNrMatchers) { + return false; + } + } + } while (true); + } +} Index: src/java/org/apache/lucene/search/BooleanClause.java =================================================================== --- src/java/org/apache/lucene/search/BooleanClause.java (revision 751749) +++ src/java/org/apache/lucene/search/BooleanClause.java (working copy) @@ -19,7 +19,7 @@ * limitations under the License. */ -/** A clause in a BooleanQuery. */ +/** A query clause in a BooleanQuery. */ public class BooleanClause implements java.io.Serializable { /** Specifies how clauses are to occur in matching documents. */ @@ -36,6 +36,10 @@ return ""; } + public int hashCode() { + return (Occur.MUST.equals(this)?1:0) ^ (Occur.MUST_NOT.equals(this)?2:0); + } + /** Use this operator for clauses that must appear in the matching documents. */ public static final Occur MUST = new Occur("MUST"); /** Use this operator for clauses that should appear in the @@ -105,7 +109,7 @@ /** Returns a hash code value for this object.*/ public int hashCode() { - return query.hashCode() ^ (Occur.MUST.equals(occur)?1:0) ^ (Occur.MUST_NOT.equals(occur)?2:0); + return query.hashCode() ^ occur.hashCode(); } Index: src/java/org/apache/lucene/search/ReqExclScorer.java =================================================================== --- src/java/org/apache/lucene/search/ReqExclScorer.java (revision 751749) +++ src/java/org/apache/lucene/search/ReqExclScorer.java (working copy) @@ -20,32 +20,31 @@ import java.io.IOException; -/** A Scorer for queries with a required subscorer and an excluding (prohibited) subscorer. - *
- * This Scorer implements {@link Scorer#skipTo(int)}, - * and it uses the skipTo() on the given scorers. +/** A Scorer for queries with a required scorer + * and an excluding (prohibited) DocIdSetIterator. */ -public class ReqExclScorer extends Scorer { - private Scorer reqScorer, exclScorer; +class ReqExclScorer extends Scorer { + private Scorer reqScorer; + private DocIdSetIterator exclDisi; /** Construct a ReqExclScorer. * @param reqScorer The scorer that must match, except where - * @param exclScorer indicates exclusion. + * @param exclDisi indicates exclusion. */ public ReqExclScorer( Scorer reqScorer, - Scorer exclScorer) { + DocIdSetIterator exclDisi) { super(null); // No similarity used. this.reqScorer = reqScorer; - this.exclScorer = exclScorer; + this.exclDisi = exclDisi; } private boolean firstTime = true; public boolean next() throws IOException { if (firstTime) { - if (! exclScorer.next()) { - exclScorer = null; // exhausted at start + if (! exclDisi.next()) { + exclDisi = null; // exhausted at start } firstTime = false; } @@ -56,7 +55,7 @@ reqScorer = null; // exhausted, nothing left return false; } - if (exclScorer == null) { + if (exclDisi == null) { return true; // reqScorer.next() already returned true } return toNonExcluded(); @@ -66,7 +65,7 @@ *
On entry: *

@@ -74,17 +73,17 @@ * @return true iff there is a non excluded required doc. */ private boolean toNonExcluded() throws IOException { - int exclDoc = exclScorer.doc(); + int exclDoc = exclDisi.doc(); do { int reqDoc = reqScorer.doc(); // may be excluded if (reqDoc < exclDoc) { - return true; // reqScorer advanced to before exclScorer, ie. not excluded + return true; // reqScorer advanced to before exclDisi, ie. not excluded } else if (reqDoc > exclDoc) { - if (! exclScorer.skipTo(reqDoc)) { - exclScorer = null; // exhausted, no more exclusions + if (! exclDisi.skipTo(reqDoc)) { + exclDisi = null; // exhausted, no more exclusions return true; } - exclDoc = exclScorer.doc(); + exclDoc = exclDisi.doc(); if (exclDoc > reqDoc) { return true; // not excluded } @@ -115,14 +114,14 @@ public boolean skipTo(int target) throws IOException { if (firstTime) { firstTime = false; - if (! exclScorer.skipTo(target)) { - exclScorer = null; // exhausted + if (! exclDisi.skipTo(target)) { + exclDisi = null; // exhausted } } if (reqScorer == null) { return false; } - if (exclScorer == null) { + if (exclDisi == null) { return reqScorer.skipTo(target); } if (! reqScorer.skipTo(target)) { @@ -134,7 +133,7 @@ public Explanation explain(int doc) throws IOException { Explanation res = new Explanation(); - if (exclScorer.skipTo(doc) && (exclScorer.doc() == doc)) { + if (exclDisi.skipTo(doc) && (exclDisi.doc() == doc)) { res.setDescription("excluded"); } else { res.setDescription("not excluded"); @@ -142,4 +141,8 @@ } return res; } + + public String toString() { + return "ReqExclScorer(required scorer: " + reqScorer + ", prohibited disi: " + exclDisi + ")"; + } } Index: src/java/org/apache/lucene/search/BooleanFilterClause.java =================================================================== --- src/java/org/apache/lucene/search/BooleanFilterClause.java (revision 0) +++ src/java/org/apache/lucene/search/BooleanFilterClause.java (revision 0) @@ -0,0 +1,84 @@ +package org.apache.lucene.search; + +import org.apache.lucene.util.Parameter; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** A filter clause in a BooleanQuery. + A Filter may only occur as MUST or MUST_NOT. + */ +public class BooleanFilterClause implements java.io.Serializable { + + /** The filter whose matching documents are combined by the boolean query. */ + private Filter filter; + + private BooleanClause.Occur occur; // Could move to superclass + + /** Constructs a BooleanFilterClause. + */ + public BooleanFilterClause(Filter filter, BooleanClause.Occur occur) { + if (occur == BooleanClause.Occur.SHOULD) { + throw new IllegalArgumentException("Filter can only occur as MUST or MUST_NOT"); + } + this.filter = filter; + this.occur = occur; + } + + public BooleanClause.Occur getOccur() { + return occur; + } + + public void setOccur(BooleanClause.Occur occur) { + if (occur == BooleanClause.Occur.SHOULD) { + throw new IllegalArgumentException("Filter can only occur as MUST or MUST_NOT"); + } + this.occur = occur; + } + + public Filter getFilter() { + return filter; + } + + public void setFilter(Filter filter) { + assert filter == null; + this.filter = filter; + } + + public boolean isRequired() { + return BooleanClause.Occur.MUST.equals(occur); + } + + /** Returns true iff o is equal to this. */ + public boolean equals(Object o) { + if (!(o instanceof BooleanFilterClause)) { + return false; + } + BooleanFilterClause other = (BooleanFilterClause)o; + return this.occur.equals(other.occur) + && this.filter.equals(other.filter); + } + + /** Returns a hash code value for this object.*/ + public int hashCode() { + return filter.hashCode() ^ occur.hashCode(); + } + + public String toString() { + return occur.toString() + filter.toString(); + } +} Index: src/java/org/apache/lucene/util/DisiDocQueue.java =================================================================== --- src/java/org/apache/lucene/util/DisiDocQueue.java (revision 0) +++ src/java/org/apache/lucene/util/DisiDocQueue.java (revision 0) @@ -0,0 +1,206 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Derived from org.apache.lucene.util.ScorerDocQueue of July 2008 */ + +import java.io.IOException; +import org.apache.lucene.search.DocIdSetIterator; + +/** A DisiDocQueue maintains a partial ordering of its DocIdSetIterators such that the + * least DocIdSetIterator (disi) can always be found in constant time. + * Put()'s and pop()'s require log(size) time. + * The ordering is by DocIdSetIterator.doc(). + */ +public class DisiDocQueue { + private final HeapedDisiDoc[] heap; + private final int maxSize; + private int size; + + private class HeapedDisiDoc { + DocIdSetIterator disi; + int doc; + + HeapedDisiDoc(DocIdSetIterator disi) { this(disi, disi.doc()); } + + HeapedDisiDoc(DocIdSetIterator disi, int doc) { + this.disi = disi; + this.doc = doc; + } + + void adjust() { doc = disi.doc(); } + } + + private HeapedDisiDoc topHDD; // same as heap[1], only for speed + + /** Create a DisiDocQueue with a maximum size. */ + public DisiDocQueue(int maxSize) { + // assert maxSize >= 0; + size = 0; + int heapSize = maxSize + 1; + heap = new HeapedDisiDoc[heapSize]; + this.maxSize = maxSize; + topHDD = heap[1]; // initially null + } + + /** + * Adds a DocIdSetIterator to a DisiDocQueue in log(size) time. + * If one tries to add more DocIdSetIterators than maxSize + * a RuntimeException (ArrayIndexOutOfBound) is thrown. + */ + public final void put(DocIdSetIterator disi) { + size++; + heap[size] = new HeapedDisiDoc(disi); + upHeap(); + } + + /** + * Adds a DocIdSetIterator to the DisiDocQueue in log(size) time if either + * the DisiDocQueue is not full, or not lessThan(disi, top()). + * @param disi + * @return true if DocIdSetIterator is added, false otherwise. + */ + public boolean insert(DocIdSetIterator disi){ + if (size < maxSize) { + put(disi); + return true; + } else { + int docNr = disi.doc(); + if ((size > 0) && (! (docNr < topHDD.doc))) { // heap[1] is top() + heap[1] = new HeapedDisiDoc(disi, docNr); + downHeap(); + return true; + } else { + return false; + } + } + } + + /** Returns the least DocIdSetIterator of the DisiDocQueue in constant time. + * Should not be used when the queue is empty. + */ + public final DocIdSetIterator top() { + return topHDD.disi; + } + + /** Returns document number of the least Scorer of the ScorerDocQueue + * in constant time. + * Should not be used when the queue is empty. + */ + public final int topDoc() { + return topHDD.doc; + } + + public final boolean topNextAndAdjustElsePop() throws IOException { + return checkAdjustElsePop( topHDD.disi.next()); + } + + public final boolean topSkipToAndAdjustElsePop(int target) throws IOException { + return checkAdjustElsePop( topHDD.disi.skipTo(target)); + } + + private boolean checkAdjustElsePop(boolean cond) { + if (cond) { // see also adjustTop + topHDD.doc = topHDD.disi.doc(); + } else { // see also popNoResult + heap[1] = heap[size]; // move last to first + heap[size] = null; + size--; + } + downHeap(); + return cond; + } + + /** Removes and returns the least disi of the DisiDocQueue in log(size) + * time. + * Should not be used when the queue is empty. + */ + public final DocIdSetIterator pop() { + DocIdSetIterator result = topHDD.disi; + popNoResult(); + return result; + } + + /** Removes the least disi of the DisiDocQueue in log(size) time. + * Should not be used when the queue is empty. + */ + private final void popNoResult() { + heap[1] = heap[size]; // move last to first + heap[size] = null; + size--; + downHeap(); // adjust heap + } + + /** Should be called when the disi at top changes doc() value. + * Still log(n) worst case, but it's at least twice as fast to 
+   *  { pq.top().change(); pq.adjustTop(); }
+   *
instead of 
+   *  { o = pq.pop(); o.change(); pq.push(o); }
+   *
+ */ + public final void adjustTop() { + topHDD.adjust(); + downHeap(); + } + + /** Returns the number of disis currently stored in the DisiDocQueue. */ + public final int size() { + return size; + } + + /** Removes all entries from the DisiDocQueue. */ + public final void clear() { + for (int i = 0; i <= size; i++) { + heap[i] = null; + } + size = 0; + } + + private final void upHeap() { + int i = size; + HeapedDisiDoc node = heap[i]; // save bottom node + int j = i >>> 1; + while ((j > 0) && (node.doc < heap[j].doc)) { + heap[i] = heap[j]; // shift parents down + i = j; + j = j >>> 1; + } + heap[i] = node; // install saved node + topHDD = heap[1]; + } + + private final void downHeap() { + int i = 1; + int j = i << 1; // find smaller child + int k = j + 1; + if ((k <= size) && (heap[k].doc < heap[j].doc)) { + j = k; + } + while ((j <= size) && (heap[j].doc < heap[i].doc)) { + heap[i] = heap[j]; // shift up child + i = j; + j = i << 1; + k = j + 1; + if (k <= size && (heap[k].doc < heap[j].doc)) { + j = k; + } + } + heap[i] = topHDD; + topHDD = heap[1]; + } +} Property changes on: src/java/org/apache/lucene/util/DisiDocQueue.java ___________________________________________________________________ Added: svn:executable + * Index: src/test/org/apache/lucene/search/TestFilterClause.java =================================================================== --- src/test/org/apache/lucene/search/TestFilterClause.java (revision 0) +++ src/test/org/apache/lucene/search/TestFilterClause.java (revision 0) @@ -0,0 +1,178 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.util.SortedVIntList; +import org.apache.lucene.util.LuceneTestCase; + +public class TestFilterClause extends LuceneTestCase { + private IndexSearcher searcher; + + public static final String field = "field"; + + public void setUp() throws Exception { + super.setUp(); + RAMDirectory directory = new RAMDirectory(); + IndexWriter writer= new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); + for (int i = 0; i < docFields.length; i++) { + Document doc = new Document(); + doc.add(new Field(field, docFields[i], Field.Store.NO, Field.Index.TOKENIZED)); + writer.addDocument(doc); + } + writer.close(); + searcher = new IndexSearcher(directory); + } + + private String[] docFields = { + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3", + "w1 xx w2 yy w3", + "w1 w3 xx w2 yy w3" + }; + + private void queryTest(Query query, int[] expDocNrs) throws Exception { + CheckHits.checkHits(query, field, searcher, expDocNrs); + } + + private TermQuery makeTermQuery(String v) { + return new TermQuery(new Term(field, v)); + } + + private Filter makeFilter(int[] filteredDocs) { + return new IntsFilter(filteredDocs); + } + + private BooleanQuery makeBooleanQuery(String[] mTs, String[] sTs, String[] pTs, int[] rDs, int[] pDs) { + BooleanQuery bq = new BooleanQuery(); + if (mTs != null) { + for (int i = 0; i < mTs.length; i++) { + bq.add(makeTermQuery(mTs[i]), BooleanClause.Occur.MUST); + } + } + if (sTs != null) { + for (int i = 0; i < sTs.length; i++) { + bq.add(makeTermQuery(sTs[i]), BooleanClause.Occur.SHOULD); + } + } + if (pTs != null) { + for (int i = 0; i < pTs.length; i++) { + bq.add(makeTermQuery(pTs[i]), BooleanClause.Occur.MUST_NOT); + } + } + if (rDs != null) { + bq.add(makeFilter(rDs), BooleanClause.Occur.MUST); + } + if (pDs != null) { + bq.add(makeFilter(pDs), BooleanClause.Occur.MUST_NOT); + } + return bq; + } + + public void testQueriesFilterClauses01() throws Exception { + String[] rt = {"w3"}; + int[] exp0123 = {0,1,2,3}; + int[] exp01 = {0,1}; + int[] exp23 = {2,3}; + int[] exp2 = {2}; + int[] exp3 = {3}; + queryTest(makeBooleanQuery(rt, null, null, null, null), exp0123); + queryTest(makeBooleanQuery(rt, null, null, exp23, null), exp23); + queryTest(makeBooleanQuery(rt, null, null, null, exp23), exp01); + queryTest(makeBooleanQuery(rt, null, null, exp2, null), exp2); + queryTest(makeBooleanQuery(rt, null, null, exp23, exp2), exp3); + } + /* This query (generated by testRandomQueries) fails on 20081229 with the filter clauses added + to BooleanQuery: + -data:X data:Y -data:1 (data:B data:X -data:Z data:6) (+data:B data:Y) + fails on doc 7 of: + + String[] data = new String [] { + "A 1 2 3 4 5 6", + "Z 4 5 6", + null, + "B 2 4 5 6", + "Y 3 5 6", + null, + "C 3 6", + "X 4 5 6" + }; + + + with a positive score from the scorer, and a zero score from explanation because of prohibited X. + The scorer is wrong, there should be no match. + + Probably DisjunctionDISI is not working correctly yet. + Test DisjunctionDISI separately. + */ + public void testQueriesFilterClauses02() throws Exception { + String[] rt = {"w3"}; + String[] ot = {"w3"}; + String[] onmt = {"notThere1", "notThere2"}; + String[] pt = {"xx"}; + String[] pt2 = {"xx", "xxNotThere"}; + int[] exp01 = {0,1}; + int[] expNone = {}; + queryTest(makeBooleanQuery(rt, null, pt2, null, null), exp01); + queryTest(makeBooleanQuery(null, ot, pt2, null, null), exp01); + queryTest(makeBooleanQuery(null, ot, pt, null, null), exp01); + queryTest(makeBooleanQuery(null, onmt, pt2, null, null), expNone); + queryTest(makeBooleanQuery(null, ot, ot, null, null), expNone); + queryTest(makeBooleanQuery(rt, ot, ot, null, null), expNone); + } + +} + +class IntsFilter extends Filter { // separate class for serializability. + IntsFilter(int[] filteredDocs) { + this.filteredDocs = filteredDocs; + } + private int[] filteredDocs; + + public DocIdSet getDocIdSet(IndexReader reader) { + return new SortedVIntList(filteredDocs); + } + + public String toString() { + StringBuffer r = new StringBuffer("IntsFilter["); + for (int i = 0; i < filteredDocs.length; i++) { + r.append(filteredDocs[i]); + if (i < (filteredDocs.length-1)) { + r.append(','); + } + } + r.append(']'); + return r.toString(); + } + + public int hashCode() { + return getClass().hashCode() ^ filteredDocs.hashCode(); + } + + public boolean equals(Object other) { + return (other instanceof IntsFilter) + && filteredDocs.equals(((IntsFilter)other).filteredDocs); + } +}