Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedPriorityQueue.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedPriorityQueue.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedPriorityQueue.java (revision ) @@ -0,0 +1,259 @@ +package org.apache.lucene.search.exposed; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A PriorityQueue maintains a partial ordering of its elements such that the + * least element can always be found in constant time. Put()'s and pop()'s + * require log(size) time. + * + *

NOTE: This class pre-allocates a full array of + * length maxSize+1, in {@link #initialize}. + * + * @lucene.internal + * + * Originally taken from PriorityQueue. + * Modified to use int[] and to take a custom comparator for memory efficiency +*/ +public class ExposedPriorityQueue { + private int size; + private int maxSize; + protected int[] heap; + private ExposedComparators.OrdinalComparator comparator; + + public ExposedPriorityQueue( + ExposedComparators.OrdinalComparator comparator, int size) { + this.comparator = comparator; + initialize(size); + } + + protected boolean lessThan(int a, int b) { + return comparator.compare(a, b) < 0; + } + + /** + * This method can be overridden by extending classes to return a sentinel + * object which will be used by {@link #initialize(int)} to fill the queue, so + * that the code which uses that queue can always assume it's full and only + * change the top without attempting to insert any new object.
+ * + * Those sentinel values should always compare worse than any non-sentinel + * value (i.e., {@link #lessThan} should always favor the + * non-sentinel values).
+ * + * By default, this method returns false, which means the queue will not be + * filled with sentinel values. Otherwise, the value returned will be used to + * pre-populate the queue. Adds sentinel values to the queue.
+ * + * If this method is extended to return a non-null value, then the following + * usage pattern is recommended: + * + * 

+   * // extends getSentinelObject() to return a non-null value.
+   * PriorityQueue pq = new MyQueue(numHits);
+   * // save the 'top' element, which is guaranteed to not be null.
+   * MyObject pqTop = pq.top();
+   * <...>
+   * // now in order to add a new element, which is 'better' than top (after
+   * // you've verified it is better), it is as simple as:
+   * pqTop.change().
+   * pqTop = pq.updateTop();
+   *
+ * + * NOTE: if this method returns a non-null value, it will be called by + * {@link #initialize(int)} {@link #size()} times, relying on a new object to + * be returned and will not check if it's null again. Therefore you should + * ensure any call to this method creates a new instance and behaves + * consistently, e.g., it cannot return null if it previously returned + * non-null. + * + * @return the sentinel object to use to pre-populate the queue, or null if + * sentinel objects are not supported. + */ + protected int getSentinelObject() { + return -1; + } + + /** Subclass constructors must call this. */ + @SuppressWarnings("unchecked") + protected final void initialize(int maxSize) { + size = 0; + int heapSize; + if (0 == maxSize) + // We allocate 1 extra to avoid if statement in top() + heapSize = 2; + else { + if (maxSize == Integer.MAX_VALUE) { + // Don't wrap heapSize to -1, in this case, which + // causes a confusing NegativeArraySizeException. + // Note that very likely this will simply then hit + // an OOME, but at least that's more indicative to + // caller that this values is too big. We don't +1 + // in this case, but it's very unlikely in practice + // one will actually insert this many objects into + // the PQ: + heapSize = Integer.MAX_VALUE; + } else { + // NOTE: we add +1 because all access to heap is + // 1-based not 0-based. heap[0] is unused. + heapSize = maxSize + 1; + } + } + heap = new int[heapSize]; + this.maxSize = maxSize; + + // If sentinel objects are supported, populate the queue with them + int sentinel = getSentinelObject(); + if (sentinel != -1) { + heap[1] = sentinel; + for (int i = 2; i < heap.length; i++) { + heap[i] = getSentinelObject(); + } + size = maxSize; + } + } + + /** + * Adds an Object to a PriorityQueue in log(size) time. If one tries to add + * more objects than maxSize from initialize an + * {@link ArrayIndexOutOfBoundsException} is thrown. + * + * @return the new 'top' element in the queue. + */ + public final int add(int element) { + size++; + heap[size] = element; + upHeap(); + return heap[1]; + } + + /** + * Adds an Object to a PriorityQueue in log(size) time. + * It returns the object (if any) that was + * dropped off the heap because it was full. This can be + * the given parameter (in case it is smaller than the + * full heap's minimum, and couldn't be added), or another + * object that was previously the smallest value in the + * heap and now has been replaced by a larger one, or null + * if the queue wasn't yet full with maxSize elements. + */ + public int insertWithOverflow(int element) { + if (size < maxSize) { + add(element); + return -1; + } else if (size > 0 && !lessThan(element, heap[1])) { + int ret = heap[1]; + heap[1] = element; + updateTop(); + return ret; + } else { + return element; + } + } + + /** Returns the least element of the PriorityQueue in constant time. */ + public final int top() { + // We don't need to check size here: if maxSize is 0, + // then heap is length 2 array with both entries null. + // If size is 0 then heap[1] is already null. + return heap[1]; + } + + /** Removes and returns the least element of the PriorityQueue in log(size) + time. */ + public final int pop() { + if (size > 0) { + int result = heap[1]; // save first value + heap[1] = heap[size]; // move last to first + heap[size] = -1; // permit GC of objects + size--; + downHeap(); // adjust heap + return result; + } else + return -1; + } + + /** + * Should be called when the Object at top changes values. Still log(n) worst + * case, but it's at least twice as fast to + * + * 
+   * pq.top().change();
+   * pq.updateTop();
+   *
+ * + * instead of + * + * 
+   * o = pq.pop();
+   * o.change();
+   * pq.push(o);
+   *
+ * + * @return the new 'top' element. + */ + public final int updateTop() { + downHeap(); + return heap[1]; + } + + /** Returns the number of elements currently stored in the PriorityQueue. */ + public final int size() { + return size; + } + + /** Removes all entries from the PriorityQueue. */ + public final void clear() { + for (int i = 0; i <= size; i++) { + heap[i] = -1; + } + size = 0; + } + + private final void upHeap() { + int i = size; + int node = heap[i]; // save bottom node + int j = i >>> 1; + while (j > 0 && lessThan(node, heap[j])) { + heap[i] = heap[j]; // shift parents down + i = j; + j = j >>> 1; + } + heap[i] = node; // install saved node + } + + private final void downHeap() { + int i = 1; + int node = heap[i]; // save top node + int j = i << 1; // find smaller child + int k = j + 1; + if (k <= size && lessThan(heap[k], heap[j])) { + j = k; + } + while (j <= size && lessThan(heap[j], node)) { + heap[i] = heap[j]; // shift up child + i = j; + j = i << 1; + k = j + 1; + if (k <= size && lessThan(heap[k], heap[j])) { + j = k; + } + } + heap[i] = node; // install saved node + } +} \ No newline at end of file Index: solr/contrib/exposed/src/test/resources/solr/conf/synonyms.txt =================================================================== --- solr/contrib/exposed/src/test/resources/solr/conf/synonyms.txt (revision ) +++ solr/contrib/exposed/src/test/resources/solr/conf/synonyms.txt (revision ) @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermDocIterator.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermDocIterator.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermDocIterator.java (revision ) @@ -0,0 +1,113 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Single source and single field Iterator that provides terms and potentially + * document IDs. + *

+ * It is recommended to use a {@link CachedTermProvider} as source or otherwise + * ensure that ordered iteration is fast if no docIDs are to be returned.. + * If docIDs are to be returned, caching at the source level will not help. + *

+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless + * {@link #setReuseTuple(boolean)} is called with false. + */ +public class TermDocIterator implements Iterator { + private final TermProvider source; + private final PackedInts.Reader order; + private final boolean collectDocIDs; + + private int position = 0; + private boolean reuseTuple = true; + + private DocsEnum docsEnum = null; // Reusing + + private boolean pending = false; // true == Terms or docIDs are ready in tuple + private final ExposedTuple tuple; + + public TermDocIterator( + TermProvider source, boolean collectDocIDs) throws IOException { + this.source = source; + this.collectDocIDs = collectDocIDs; + order = source.getOrderedOrdinals(); + String field = source.getField(0); + + tuple = new ExposedTuple(field, null, 0, 0, null, 0); + } + + // Ensure that we can deliver an id + public boolean hasNext() { + while (true) { + if (pending) { + return true; + } + if (position >= order.size()) { + if (source instanceof CachedTermProvider && ExposedSettings.debug) { + System.out.println("TermDocsIterator depleted: " + + ((CachedTermProvider)source).getStats()); + } + return false; + } + try { + seekToNextTerm(); + } catch (IOException e) { + throw new RuntimeException( + "IOException while seeking to next docID", e); + } + } + } + + public ExposedTuple next() { + if (!hasNext()) { + throw new IllegalStateException("The iterator is depleted"); + } + pending = false; + return reuseTuple ? tuple : new ExposedTuple(tuple); + } + + private void seekToNextTerm() throws IOException { + while (true) { + pending = false; + if (position >= order.size()) { + return; + } + tuple.indirect = position; + tuple.ordinal = order.get(position); + try { + tuple.term = source.getTerm(tuple.ordinal); + } catch (IOException e) { + throw new RuntimeException( + "Unable to resolve the term at order=" + + order + ", ordinal=" + order.get(position), e); + } + position++; + + if (!collectDocIDs) { + pending = true; + return; + } + + // TODO: Speed up by reusing (this currently breaks TestExposedCache.testIndirectIndex) + docsEnum = source.getDocsEnum(tuple.ordinal, null); + if (docsEnum == null) { + continue; // TODO: Is this possible? + } + tuple.docIDs = docsEnum; + pending = true; + return; + } + } + + public void remove() { + throw new UnsupportedOperationException("Not a valid operation"); + } + + public void setReuseTuple(boolean reuseTuple) { + this.reuseTuple = reuseTuple; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java (revision ) @@ -0,0 +1,157 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; + +/** + * Wrapper for TermProvider that provides a flexible cache for terms. + * Warning: This cache does not work well for docIDs. + *

+ * For random access, a plain setup with no read-ahead is suitable. + *

+ * For straight iteration in ordinal order, a setup with maximum read-ahead + * is preferable. This is uncommon. + *

+ * For iteration in sorted ordinal order, where the ordinal order is fairly + * aligned to unicode sorting order, a setup with some read-ahead works well. + * This is a common case. + *

+ * For merging chunks (See the SegmentReader.getSortedTermOrdinals), where + * the ordinal order inside the chunks if fairly aligned to unicode sorting + * order, a read-ahead works iff {@link #onlyReadAheadIfSpace} is true as this + * prevents values from the beginning of other chunks from being flushed when + * values from the current chunk is filled with read-ahead. This requires that + * the merger removes processed values from the cache explicitly. + *

+ * If the underlying order is used by calling {@link #getOrderedTerm(long)}, + * {@link #getOrderedField(long)} or {@link #getOrderedOrdinals()}, the + * methods are delegated directly to the backing TermProvider. It is the + * responsibility of the caller to endure that proper caching of the order + * is done at the source level. + */ +public class CachedTermProvider extends CachedProvider + implements TermProvider { + private final TermProvider source; + + /** + * + * @param source the backing term provider. + * @param cacheSize the maximum number of elements to hold in cache. + * @param readAhead the maximum number of lookups that can be performed + * after a plain lookup. + * @throws java.io.IOException if the cache could access the source. + */ + public CachedTermProvider( + TermProvider source, int cacheSize, int readAhead) throws IOException { + super(cacheSize, readAhead, source.getOrdinalTermCount()-1); + this.source = source; + } + + @Override + protected BytesRef lookup(final long index) throws IOException { + return source.getTerm(index); + } + + @Override + public BytesRef getTerm(final long ordinal) throws IOException { + return get(ordinal); + } + + @Override + public Iterator getIterator(boolean collectDocIDs) + throws IOException { + throw new UnsupportedOperationException( + "The cache does not support the creation of iterators"); + } + + @Override + public String getDesignation() { + return "CachedTermProvider(" + source.getClass().getSimpleName() + "(" + + source.getDesignation() + "))"; + } + + public void transitiveReleaseCaches(int level, boolean keepRoot) { + clear(); + source.transitiveReleaseCaches(level, keepRoot); + } + + /* Straight delegations */ + + public int getNearestTermIndirect(BytesRef key) throws IOException { + return source.getNearestTermIndirect(key); + } + + public int getNearestTermIndirect(BytesRef key, int startTermPos, int endTermPos) throws IOException { + return source.getNearestTermIndirect(key, startTermPos, endTermPos); + } + + public Comparator getComparator() { + return source.getComparator(); + } + + public String getComparatorID() { + return source.getComparatorID(); + } + + public String getField(long ordinal) throws IOException { + return source.getField(ordinal); + } + + public String getOrderedField(long indirect) throws IOException { + return source.getOrderedField(indirect); + } + + public BytesRef getOrderedTerm(long indirect) throws IOException { + return source.getOrderedTerm(indirect); + } + + public long getUniqueTermCount() throws IOException { + return source.getUniqueTermCount(); + } + + public long getOrdinalTermCount() throws IOException { + return source.getOrdinalTermCount(); + } + + public long getMaxDoc() { + return source.getMaxDoc(); + } + + public IndexReader getReader() { + return source.getReader(); + } + + public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException { + return source.getDocsEnum(ordinal, reuse); + } + + public PackedInts.Reader getOrderedOrdinals() throws IOException { + return source.getOrderedOrdinals(); + } + + public PackedInts.Reader getDocToSingleIndirect() throws IOException { + return source.getDocToSingleIndirect(); + } + + public int getReaderHash() { + return source.getReaderHash(); + } + + public int getRecursiveHash() { + return source.getRecursiveHash(); + } + + public int getDocIDBase() { + return source.getDocIDBase(); + } + + public void setDocIDBase(int base) { + source.setDocIDBase(base); + } +} \ No newline at end of file Index: solr/contrib/exposed/src/test/java/org/apache/solr/exposed/TestExposedFacetQueryComponent.java =================================================================== --- solr/contrib/exposed/src/test/java/org/apache/solr/exposed/TestExposedFacetQueryComponent.java (revision ) +++ solr/contrib/exposed/src/test/java/org/apache/solr/exposed/TestExposedFacetQueryComponent.java (revision ) @@ -0,0 +1,372 @@ +package org.apache.solr.exposed; + +import org.apache.lucene.index.Term; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.update.processor.LogUpdateProcessorFactory; +import org.apache.solr.update.processor.UpdateRequestProcessor; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Locale; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.apache.solr.exposed.ExposedFacetParams.*; + +public class TestExposedFacetQueryComponent extends SolrTestCaseJ4 { + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-exposed.xml","schema-exposed.xml"); + } + + @After + @Override + public void tearDown() throws Exception { + close(); + super.tearDown(); + } + + String t(int tnum) { + return String.format(Locale.US, "%08d", tnum); + } + + void createIndex(int nTerms) { + assertU(delQ("*:*")); + for (int i=0; i10")); + +// System.out.println(response.replace("><", ">\n<")); + } + + public void testPathField() throws Exception { + addHierarchicalContent(100, 3, 3); + String response = h.query(req( + "qt", "exprh", + "q", "*:*", + EFACET, "true", + EFACET_MINCOUNT, "1", + EFACET_FIELD, "path_ss")); + + assertTrue("The facet response '" + response + + "' should contain tags for field 'path'", + response.contains("")); + +// System.out.println(response.replace("><", ">\n<")); + } + + @Test + public void testSorted() throws Exception { + addContent(100); + String response = h.query(req( + "qt", "exprh", + "q", "modulo2_s:mod1", + EFACET, "true", + EFACET_MINCOUNT, "1", + EFACET_SORT, EFACET_SORT_INDEX, + EFACET_FIELD, "modulo5_s")); + + int previousPos = -1; + for (int i = 0 ; i < 5 ; i++) { + String mod = "mod" + i; + int newPos = response.indexOf(""); + assertTrue("The index for '" + mod + "' should be larger than the " + + "previous index " + previousPos + " but was " + newPos, + previousPos < newPos); + previousPos = newPos; + } + } + + public void testHierarchical() throws Exception { + addHierarchicalContent(100, 3, 3); + String response = h.query(req( + "qt", "exprh", + "q", "*:*", + EFACET, "true", + EFACET_MINCOUNT, "1", + EFACET_HIERARCHICAL, "true", + EFACET_LIMIT, "2", + EFACET_HIERARCHICAL_LEVELS, "2", + EFACET_FIELD, "path_ss")); + +/* assertTrue("The facet response '" + response + + "' should contain tags for field 'path'", + response.contains("")); + */ + System.out.println(response.replace("><", ">\n<")); + } + + public void testGen() { + addHierarchicalContent(2, new int[]{1, 2}, new int[]{2, 3}); + } + + // We have a problem here: Document per document index update with adoc is + // too slow for proper scale testing + public void disabledtestSmallScalePerformance() throws Exception { +// System.out.println( +// "Changing level to SEVERE for Solrcore and UpdateRequestProcessor"); + // Logging each update is a major performance drain + java.util.logging.Logger.getLogger(SolrCore.log.getName()). + setLevel(Level.SEVERE); + java.util.logging.Logger.getLogger(UpdateRequestProcessor.class.getName()). + setLevel(Level.SEVERE); + + long buildTime = -System.currentTimeMillis(); + addHierarchicalContent(2600, new int[]{1, 1}, new int[]{26, 100}); + buildTime += System.currentTimeMillis(); + + long facetTime = -System.currentTimeMillis(); + String response = h.query(req( + "qt", "exprh", + "q", "*:*", + "indent", "on", + EFACET, "true", + EFACET_MINCOUNT, "1", + EFACET_HIERARCHICAL, "true", + EFACET_LIMIT, "10", + EFACET_HIERARCHICAL_LEVELS, "2", + EFACET_FIELD, "path_ss")); + facetTime += System.currentTimeMillis(); + System.out.println(response); + System.out.println( + "Build time: " + buildTime + "ms, facet time: " + facetTime + "ms"); + } + + // Mimics facet_samples.tcl from SOLR-2412 + + private void addHierarchicalContent(int docs, int[] elements, int[] uniques) { + int[] countersExp = new int[elements.length]; + Arrays.fill(countersExp, 1); + int[] countersPivot = new int[elements.length]; + Arrays.fill(countersPivot, 1); + + if (uniques == null) { + uniques = new int[elements.length]; + Arrays.fill(uniques, Integer.MAX_VALUE); + } + + int paths = 1; + int levels = 0; + for (int element: elements) { + paths *= element; // exp + levels += element; + } + String s = paths == 1 ? "_s" : "_ss"; + String pathHeader = "path" + s; + + String[] values = new String[2 + (paths + levels) * 2 ]; + int pos = 0; + values[pos++] = "id"; values[pos++] = "dummy"; + // path_s + for (int heading = 0 ; heading < paths ; heading++) { + values[pos++] = pathHeader; values[pos++] = "dummy"; + } + // level_s + for (int level = 0 ; level < elements.length ; level++) { + for (int i = 0 ; i < elements[level] ; i++) { + values[pos++] = "level" + level + s; values[pos++] = "dummy"; + } + } + + long genTime = 0; + long addTime = 0; + + for (int docID = 1 ; docID <= docs ; docID++) { + genTime -= System.nanoTime(); + values[1] = Integer.toString(docID); + pos = addExp(values, elements, uniques, countersExp); + addPivot(values, pos, elements, uniques, countersPivot); + genTime += System.nanoTime(); + + addTime -= System.nanoTime(); + assertU(adoc(values)); + addTime += System.nanoTime(); +/* for (String val: values) { + System.out.print(" " + val); + } + System.out.println("");*/ + if (docID >>> 10 << 10 == docID) { + System.out.println(docID + "/" + docs + ". Gentime: " + genTime/1000000 + + "ms, addtime: " + addTime/1000000 + "ms"); + } + } + assertU(commit()); + } + + private int addExp(String[] values, + int[] elements, int[] uniques, int[] counters) { + return addExp(values, 3, "", 0, elements, uniques, counters); + } + private int addExp(String[] values, int pos, String path, int level, + int[] elements, int[] uniques, int[] counters) { + if (level == elements.length) { + values[pos] = path; + return pos+2; + } + if (level > 0) { + path = path + "/"; + } + for (int e = 0 ; e < elements[level] ; e++) { + String val = "L" + level + "_T" + incGet(counters, uniques, level); + pos = addExp( + values, pos, path + val, level+1, elements, uniques, counters); + } + return pos; + } + + private void addPivot(String[] values, int pos, + int[] elements, int[] uniques, int[] counters) { + int combos = 1; + for (int level = 0 ; level < elements.length ; level++) { + int c = elements[level]; + combos *= c; + String path = "L" + level + "_T"; + for (int tag = 0 ; tag < combos ; tag++) { + values[pos] = path + incGet(counters, uniques, level); + pos += 2; + } + } + } + + private int incGet(int[] counters, int[] uniques, int level) { + final int result = counters[level]; + counters[level]++; + if (counters[level] > uniques[level]) { + counters[level] = 1; + } + return result; + } + + public void testSort() throws Exception { + assertU(adoc("id", "1", + "level1_s", "1A", + "level2_ss", "2A", + "level2_ss", "2B" + )); + assertU(adoc("id", "2", + "level1_s", "2A", + "level2_ss", "2A", + "level2_ss", "2B", + "level2_ss", "2C" + )); + assertU(commit()); + String response = h.query(req( + "q", "*:*", + "indent", "on", + "rows", "1", + "fl", "id", + "facet", "on", + "facet.pivot", "level1_s,level2_ss", + "facet.limit", "1", + "facet.sort", "count" + )); + System.out.println(response); + } + + + // The TagExtractor used optimized extraction of relevant Tags when + // non-count-order is used. This code needs to be updated to support reverse +/* @Test + public void testSortedReverse() throws Exception { + addContent(100); + String response = h.query(req( + "qt", "exprh", + "q", "modulo2_s:mod1", + EFACET, "true", + EFACET_MINCOUNT, "1", + EFACET_SORT, EFACET_SORT_INDEX, + EFACET_REVERSE, "true", + EFACET_FIELD, "modulo5_s")); + + int previousPos = Integer.MAX_VALUE; + for (int i = 0 ; i < 5 ; i++) { + String mod = "mod" + i; + int newPos = response.indexOf(""); + assertTrue("The index for '" + mod + "' should be smaller than the " + + "previous index " + previousPos + " but was " + newPos, + previousPos > newPos); + previousPos = newPos; + } + }*/ + + private void addContent(int num) { + for (int id = 0 ; id < num ; id++) { + assertU(adoc("id", Integer.toString(id), + "number_s", "num" + Integer.toString(id), + "text", Integer.toString(id), + "many_ws", Integer.toString(id), + "modulo2_s", "mod" + Integer.toString(id % 2), + "modulo5_s", "mod" + Integer.toString(id % 5))); + } + assertU(commit()); + } + + /** + * @param docs the number of documents to add. + * @param paths the number of paths for each document. + * @param depth the depth of the paths to add. Last entry in the hierarchy + * will be the document id. + */ + private void addHierarchicalContent(int docs, int paths, int depth) { + StringBuffer sb = new StringBuffer(100); + for (int doc = 0 ; doc < docs ; doc++) { + ArrayList content = new ArrayList(2 + paths * 2); + content.add("id"); + content.add(Integer.toString(doc)); + for (int path = 0 ; path < paths ; path++) { + sb.setLength(0); + for (int d = 0 ; d < depth ; d++) { + if (d != 0) { + sb.append("/"); + } + // 0_0/0_1/0_2 + // 1_0/1_1/1_2 + if (d == depth-1) { + sb.append(doc); + } else { + sb.append(path).append("_").append(d); + } + } + content.add("path_ss"); + content.add(sb.toString()); + } + content.add("modulo2_s"); + content.add("mod" + Integer.toString(doc % 2)); + content.add("modulo5_s"); + content.add("mod" + Integer.toString(doc % 5)); + String[] array = new String[content.size()]; + array = content.toArray(array); + assertU(adoc(array)); + } + assertU(commit()); + } +} Index: solr/example/solr/conf/solrconfig.xml =================================================================== --- solr/example/solr/conf/solrconfig.xml (revision 1142722) +++ solr/example/solr/conf/solrconfig.xml (revision ) @@ -1579,4 +1579,18 @@ --> + + + 10 + 5 + 5 + + + + + + exposedComponent + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xml =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xml (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xml (revision ) @@ -0,0 +1,43 @@ + + + + freetext:"foo <> bar & zoo" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedUtil.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedUtil.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedUtil.java (revision ) @@ -0,0 +1,15 @@ +package org.apache.lucene.search.exposed; + +public class ExposedUtil { + private ExposedUtil() { + // Disables all constructors + } + + public static String time(String type, long num, long ms) { + if (ms == 0) { + return num + " " + type + " in 0 ms"; + } + return num + " " + type + " in " + ms + " ms: ~= " + + num/ms + " " + type + "/ms"; + } +} Index: solr/contrib/exposed/src/test/resources/solr/conf/schema-exposed.xml =================================================================== --- solr/contrib/exposed/src/test/resources/solr/conf/schema-exposed.xml (revision ) +++ solr/contrib/exposed/src/test/resources/solr/conf/schema-exposed.xml (revision ) @@ -0,0 +1,364 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + text + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagExtractor.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagExtractor.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagExtractor.java (revision ) @@ -0,0 +1,322 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.search.exposed.ExposedComparators; +import org.apache.lucene.search.exposed.ExposedPriorityQueue; +import org.apache.lucene.search.exposed.TermProvider; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.search.exposed.facet.request.FacetRequestGroup; +import org.apache.lucene.search.exposed.facet.request.SubtagsConstraints; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Helper class for extracting tags for a faceting structure. + */ +public class TagExtractor { + final private FacetRequestGroup requestGroup; + final private Pattern splitPattern; // Defined if hierarchical + + public TagExtractor(FacetRequestGroup requestGroup) { + this.requestGroup = requestGroup; + splitPattern = requestGroup.isHierarchical() ? + Pattern.compile(requestGroup.getDelimiter()) : + null; + } + + public FacetResponse.Group extract( + int groupID, FacetMap map, + int[] tagCounts, int startPos, int endPos) throws IOException { + + if (requestGroup.isHierarchical()) { + TermProvider provider = map.getProviders().get(groupID); + if (!(provider instanceof HierarchicalTermProvider)) { + throw new IllegalStateException( + "Internal inconsistency: The provider for " + + requestGroup.getGroup().getName() + + " should be hierarchical" + + " but is " + provider.getClass()); + } + int delta = -map.getIndirectStarts()[groupID]; + return new FacetResponse.Group( + requestGroup, + extractHierarchical(requestGroup.getDeeperLevel(), 1, + (HierarchicalTermProvider)provider, delta, + map, tagCounts, startPos, endPos)); + } + + FacetRequest.GROUP_ORDER order = requestGroup.getOrder(); + if (FacetRequest.GROUP_ORDER.count == order) { + return extractCountResult( + requestGroup, map, tagCounts, startPos, endPos); + } else if (FacetRequest.GROUP_ORDER.index == order + || FacetRequest.GROUP_ORDER.locale == order) { + return extractOrderResult( + groupID, map, tagCounts, startPos, endPos); + } + throw new UnsupportedOperationException( + "The order '" + order + "' is unknown"); + } + + private FacetResponse.TagCollection extractHierarchical( + final SubtagsConstraints constraints, + final int currentLevel, + final HierarchicalTermProvider provider, final int delta, + final FacetMap map, + final int[] tagCounts, final int startPos, final int endPos) + throws IOException { + if (currentLevel > requestGroup.getLevels()) { + return null; // Stop descending + } + // TODO: Support count, offset, path etc. + switch( constraints.getSubtagsOrder()) { + case base: return extractHierarchicalOrder( + constraints, currentLevel, provider, delta, map, + tagCounts, startPos, endPos); + case count: return extractHierarchicalCount( + constraints, currentLevel, provider, delta, map, + tagCounts, startPos, endPos); + default: throw new IllegalArgumentException( + "The order '" + constraints.getSubtagsOrder() + "' is unknown"); + } + } + + private class HCElement { + final int tagStartPos; + final int tagEndPos; + final int count; + final int totalCount; + + private HCElement(int tagStartPos, int tagEndPos, int count, int totalCount) { + this.tagStartPos = tagStartPos; + this.tagEndPos = tagEndPos; + this.count = count; + this.totalCount = totalCount; + } + } + + private class HCEPQ extends PriorityQueue { + private HCEPQ(int size) { + super(size, false); // TODO: Consider prepopulating + } + + @Override + protected boolean lessThan(HCElement a, HCElement b) { + return a.totalCount == b.totalCount ? + a.tagStartPos > b.tagStartPos : + a.totalCount < b.totalCount; + } + } + + private FacetResponse.TagCollection extractHierarchicalCount( + final SubtagsConstraints constraints, final int level, + final HierarchicalTermProvider provider, final int delta, + final FacetMap map, + final int[] tagCounts, final int startPos, final int endPos) + throws IOException { + HCEPQ pq = new HCEPQ(Math.min(constraints.getMaxTags(), endPos - startPos)); + long validTags = 0; + long totalCount = 0; + long count = 0; + { // Find most popular tags + final TagSumIterator tagIterator = new TagSumIterator( + provider, constraints, + tagCounts, startPos, endPos, level, delta); + while (tagIterator.next()) { + totalCount += tagIterator.getTotalCount(); + count += tagIterator.getCount(); + final HCElement current = new HCElement( + tagIterator.tagStartPos, tagIterator.tagEndPos, + tagIterator.count, tagIterator.totalCount); + pq.insertWithOverflow(current); // TODO: Consider reusing objects + validTags++; + } + } + // Build result structure and + FacetResponse.Tag[] tags = new FacetResponse.Tag[pq.size()]; + for (int i = pq.size()-1 ; i >= 0 ; i--) { + HCElement element = pq.pop(); + String term = provider.getOrderedTerm(element.tagStartPos + delta). + utf8ToString(); + FacetResponse.Tag tag = new FacetResponse.Tag( + getLevelTerm(term, level), element.count, element.totalCount); + tags[i] = tag; + if (level < requestGroup.getLevels() && + !(provider.getLevel(element.tagStartPos) < level+1 && + element.tagStartPos+1 == element.tagEndPos)) { + tag.setSubTags(extractHierarchical( + constraints.getDeeperLevel(), level+1, provider, delta, map, + tagCounts, element.tagStartPos, element.tagEndPos)); + } + // TODO: State totalcount in tags so they are not forgotten + } + FacetResponse.TagCollection collection = new FacetResponse.TagCollection(); + collection.setTags(Arrays.asList(tags)); + collection.setTotalCount(totalCount); + collection.setTotalTags(validTags); + collection.setCount(count); + collection.setDefiner(constraints); + collection.setPotentialTags(endPos-startPos); + return collection; + } + + private FacetResponse.TagCollection extractHierarchicalOrder( + final SubtagsConstraints constraints, + final int level, + final HierarchicalTermProvider provider, + final int delta, final FacetMap map, + final int[] tagCounts, final int startTermPos, final int endTermPos) + throws IOException { + final TagSumIterator tagIterator = new TagSumIterator( + provider, constraints, + tagCounts, startTermPos, endTermPos, level, delta); + // TODO: Consider optimization by not doing totalTags, count and totalCount + List tags = new ArrayList( + Math.max(1, Math.min(constraints.getMaxTags(), 100000))); + long validTags = 0; + long totalCount = 0; + long count = 0; + while (tagIterator.next()) { + totalCount += tagIterator.getTotalCount(); + count += tagIterator.getCount(); + if (validTags < constraints.getMaxTags()) { + String term = provider.getOrderedTerm(tagIterator.tagStartPos + delta). + utf8ToString(); + FacetResponse.Tag tag = + new FacetResponse.Tag(getLevelTerm(term, level), + tagIterator.getCount(), tagIterator.getTotalCount()); + tags.add(tag); + if (level < requestGroup.getLevels() && + !(provider.getLevel(tagIterator.tagStartPos) < level+1 && + tagIterator.tagStartPos+1 == tagIterator.tagEndPos)) { + tag.setSubTags(extractHierarchical( + constraints.getDeeperLevel(), level+1, provider, delta, map, + tagCounts, tagIterator.tagStartPos, tagIterator.tagEndPos)); + } + } + validTags++; + } + FacetResponse.TagCollection collection = new FacetResponse.TagCollection(); + collection.setTags(tags); + collection.setTotalCount(totalCount); + collection.setTotalTags(validTags); + collection.setCount(count); + collection.setDefiner(constraints); + collection.setPotentialTags(endTermPos-startTermPos); + return collection; + } + + private String getLevelTerm(String term, int level) { + try { + return splitPattern.split(term)[level-1]; + } catch (ArrayIndexOutOfBoundsException e) { + ArrayIndexOutOfBoundsException ex = new ArrayIndexOutOfBoundsException( + "Unable to split '" + term + "' with delimiter '" + + splitPattern.pattern() + "' into " + level + " parts or more"); + ex.initCause(e); + throw ex; + } + } + + private FacetResponse.Group extractOrderResult( + final int groupID, final FacetMap map, + final int[] tagCounts, final int startTermPos, final int endTermPos) + throws IOException { + long extractionTime = System.currentTimeMillis(); + // Locate prefix + int origo = startTermPos; + if (requestGroup.getPrefix() != null + && !"".equals(requestGroup.getPrefix())) { + int tpOrigo = map.getProviders().get(groupID).getNearestTermIndirect( + new BytesRef(requestGroup.getPrefix())); + origo = tpOrigo + map.getIndirectStarts()[groupID]; + //origo = origo < 0 ? (origo + 1) * -1 : origo; // Ignore missing match + } + + // TODO: Add sort order + // skip to offset (only valid) + final int minCount = requestGroup.getMinCount(); + final int direction = requestGroup.getOffset() < 0 ? -1 : 1; + int delta = Math.abs(requestGroup.getOffset()); + while (delta > 0) { + origo += direction; + if (origo < startTermPos || origo >= endTermPos) { + origo += delta-1; + break; + } + if (tagCounts[origo] >= minCount) { + delta--; + } + } + // Collect valid IDs + int collectedTags = 0; + if (origo < startTermPos) { + collectedTags = startTermPos - origo; + origo = startTermPos; + } + List tags = new ArrayList( + Math.max(0, Math.min(requestGroup.getMaxTags(), endTermPos-origo))); + for (int termPos = origo ; + termPos < endTermPos & collectedTags < requestGroup.getMaxTags() ; + termPos++) { + if (tagCounts[termPos] >= minCount) { + tags.add(new FacetResponse.Tag( + map.getOrderedTerm(termPos).utf8ToString(), tagCounts[termPos])); + collectedTags++; + } + } + FacetResponse.Group responseGroup = + new FacetResponse.Group(requestGroup, tags); + extractionTime = System.currentTimeMillis() - extractionTime; + responseGroup.setExtractionTime(extractionTime); + responseGroup.setPotentialTags(endTermPos - startTermPos); + + return responseGroup; + } + + private FacetResponse.Group extractCountResult( + FacetRequestGroup requestGroup, FacetMap map, final int[] tagCounts, + final int startTermPos, final int endTermPos) throws IOException { + long extractionTime = System.currentTimeMillis(); + // Sort tag references by count + final int maxSize = Math.min( + endTermPos- startTermPos, requestGroup.getMaxTags()); + final int minCount = requestGroup.getMinCount(); + ExposedPriorityQueue pq = new ExposedPriorityQueue( + new ExposedComparators.IndirectComparator(tagCounts), maxSize); + long totalRefs = 0; + long totalValidTags = 0; + for (int termPos = startTermPos ; termPos < endTermPos ; termPos++) { + totalRefs += tagCounts[termPos]; + if (tagCounts[termPos] >= minCount) { + pq.insertWithOverflow(termPos); + totalValidTags++; + } + } + + // Extract Tags + FacetResponse.Tag[] tags = new FacetResponse.Tag[pq.size()]; + int pos = pq.size()-1; + while (pq.size() > 0) { + final int termIndex = pq.pop(); + tags[pos--] = new FacetResponse.Tag( + map.getOrderedTerm(termIndex).utf8ToString(), tagCounts[termIndex]); + } + + // Create response + FacetResponse.Group responseGroup = + new FacetResponse.Group(requestGroup, Arrays.asList(tags)); + extractionTime = System.currentTimeMillis() - extractionTime; + responseGroup.setExtractionTime(extractionTime); + responseGroup.setPotentialTags(endTermPos-startTermPos); + responseGroup.setTotalReferences(totalRefs); + responseGroup.setValidTags(totalValidTags); + return responseGroup; + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java (revision ) @@ -0,0 +1,345 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.search.exposed.facet.request.FacetRequestGroup; +import org.apache.lucene.search.exposed.facet.request.SubtagsConstraints; + +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; +import java.io.StringWriter; +import java.util.List; + +public class FacetResponse { + public static final String NAMESPACE = + "http://lucene.apache.org/exposed/facet/response/1.0"; + + private static final XMLOutputFactory xmlOutFactory; + static { + xmlOutFactory = XMLOutputFactory.newInstance(); + } + + private final FacetRequest request; + private final List groups; + private long hits; + private long countingTime = -1; // ms: just counting the references + private long totalTime = -1; // ms + private boolean countCached = false; + + public FacetResponse(FacetRequest request, List groups, long hits) { + this.request = request; + this.groups = groups; + this.hits = hits; + } + + public void setTotalTime(long totalTime) { + this.totalTime = totalTime; + } + + public void setCountingTime(long countingTime) { + this.countingTime = countingTime; + } + + public String toXML() { + StringWriter sw = new StringWriter(); + try { + XMLStreamWriter out = xmlOutFactory.createXMLStreamWriter(sw); + out.writeStartDocument("utf-8", "1.0"); + out.writeCharacters("\n"); + toXML(out); + out.writeEndDocument(); + out.flush(); + } catch (XMLStreamException e) { // Getting here means error in the code + throw new RuntimeException("Unable to create XML", e); + } + return sw.toString(); + } + public void toXML(XMLStreamWriter out) throws XMLStreamException { + out.setDefaultNamespace(NAMESPACE); + out.writeStartElement("facetresponse"); + out.writeDefaultNamespace(NAMESPACE); + out.writeAttribute("query", request.getQuery()); + writeIfDefined(out, "hits", hits); + writeIfDefined(out, "countms", countingTime); + out.writeAttribute("countcached", countCached ? "true" : "false"); + writeIfDefined(out, "totalms", totalTime); + out.writeCharacters("\n"); + for (Group group: groups) { + group.toXML(out); + } + out.writeEndElement(); // + out.writeCharacters("\n"); + } + + public FacetRequest getRequest() { + return request; + } + + public List getGroups() { + return groups; + } + + public long getHits() { + return hits; + } + + public long getCountingTime() { + return countingTime; + } + + public long getTotalTime() { + return totalTime; + } + + public boolean isCountCached() { + return countCached; + } + + public static class Group { + private final FacetRequestGroup request; + private TagCollection tags; + private long extractionTime = -1; // ms + + public Group(FacetRequestGroup request, List tags) { + this.request = request; + this.tags = new TagCollection(); + this.tags.setTags(tags); + } + + public Group(FacetRequestGroup request, TagCollection tags) { + this.request = request; + this.tags = tags; + } + + public void toXML(XMLStreamWriter out) throws XMLStreamException { + out.writeCharacters(" "); + out.writeStartElement("facet"); + out.writeAttribute("name", request.getGroup().getName()); + out.writeAttribute("fields", getFieldsStr()); + out.writeAttribute("order", request.getOrder().toString()); + if (request.getLocale() != null) { + out.writeAttribute("locale", request.getLocale()); + } + out.writeAttribute("maxtags", Integer.toString( + request.getMaxTags() == Integer.MAX_VALUE ? -1 : + request.getMaxTags())); + writeIfDefined(out, "mincount", request.getMinCount()); + writeIfDefined(out, "offset", request.getOffset()); + if (request.isHierarchical()) { + out.writeAttribute("hierarchical", "true"); + } + // TODO: Write hierarchical attributes + writeIfDefined(out, "prefix", request.getPrefix()); + writeIfDefined(out, "extractionms", extractionTime); + + tags.toXML(out, " "); +/* writeIfDefined(out, "potentialtags", tags.getPotentialTags()); + writeIfDefined(out, "usedreferences", tags.getCount()); + writeIfDefined(out, "validtags", tags.getTotalTags()); + out.writeCharacters("\n"); + + for (Tag tag: tags.getTags()) { + tag.toXML(out); + } + */ + out.writeCharacters(" "); + out.writeEndElement(); // + out.writeCharacters("\n"); + } + + private StringBuffer sb = new StringBuffer(); + public synchronized String getFieldsStr() { + sb.setLength(0); + List fieldNames = request.getGroup().getFieldNames(); + for (int i = 0 ; i < fieldNames.size() ; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(fieldNames.get(i)); + } + return sb.toString(); + } + + public void setPotentialTags(long potentialTags) { + tags.setPotentialTags(potentialTags); + } + public void setTotalReferences(long usedReferences) { + tags.setCount(usedReferences); + } + public void setValidTags(long validTags) { + tags.setTotalTags(validTags); + } + public void setExtractionTime(long extractionTime) { + this.extractionTime = extractionTime; + } + + public FacetRequestGroup getRequest() { + return request; + } + + public TagCollection getTags() { + return tags; + } + + public long getExtractionTime() { + return extractionTime; + } + + public boolean isHierarchical() { + return request.isHierarchical(); + } + } + + public static class TagCollection { + private SubtagsConstraints constraints; + private List tags; + private long potentialTags = -1; + private long count = -1; + private long totalCount = -1; + private long totalTags = -1; + + public SubtagsConstraints getDefiner() { + return constraints; + } + public void setDefiner(SubtagsConstraints constraints) { + this.constraints = constraints; + } + public List getTags() { + return tags; + } + public void setTags(List tags) { + this.tags = tags; + } + public long getPotentialTags() { + return potentialTags; + } + public void setPotentialTags(long potentialTags) { + this.potentialTags = potentialTags; + } + public long getCount() { + return count; + } + public void setCount(long count) { + this.count = count; + } + public long getTotalCount() { + return totalCount; + } + public void setTotalCount(long totalcount) { + this.totalCount = totalcount; + } + public long getTotalTags() { + return totalTags; + } + public void setTotalTags(long totalTags) { + this.totalTags = totalTags; + } + + public SubtagsConstraints getConstraints() { + return constraints; + } + + public void toXML(XMLStreamWriter out, String prefix) + throws XMLStreamException { + writeIfDefined(out, "potentialtags", getPotentialTags()); + writeIfDefined(out, "count", getCount()); + writeIfDefined(out, "totalCount", getTotalCount()); + writeIfDefined(out, "totaltags", getTotalTags()); + if (getTags().size() > 0) { + out.writeCharacters("\n"); + } + for (Tag tag: getTags()) { + tag.toXML(out, prefix + " "); + } + } + } + + public static class Tag { + private TagCollection subTags; + private final String term; + private final int count; + private final int totalCount; + + public Tag(String term, int count) { + this.term = term; + this.count = count; + this.totalCount = -1; + } + + public Tag(String term, int count, int totalCount) { + this.term = term; + this.count = count; + this.totalCount = totalCount; + } + + public void toXML(XMLStreamWriter out, String prefix) + throws XMLStreamException { + out.writeCharacters(prefix); + out.writeStartElement("tag"); + out.writeAttribute("count", Integer.toString(count)); + if (totalCount != -1) { + out.writeAttribute("totalcount", Integer.toString(totalCount)); + } + out.writeAttribute("term", term); + if (subTags != null) { + out.writeCharacters("\n"); + out.writeCharacters(prefix); + out.writeCharacters(" "); + out.writeStartElement("subtags"); + subTags.toXML(out, prefix + " "); + out.writeCharacters(prefix); + out.writeCharacters(" "); + out.writeEndElement(); // + out.writeCharacters("\n"); + out.writeCharacters(prefix); + } + out.writeEndElement(); // + out.writeCharacters("\n"); + } + + public TagCollection getSubTags() { + return subTags; + } + + public void setSubTags(TagCollection subTags) { + this.subTags = subTags; + } + + public String getTerm() { + return term; + } + + public int getCount() { + return count; + } + + public int getTotalCount() { + return totalCount; + } + } + + private static void writeIfDefined(XMLStreamWriter out, String attributeName, + long value) throws XMLStreamException { + if (value == -1) { + return; + } + out.writeAttribute(attributeName, Long.toString(value)); + } + + private static void writeIfDefined(XMLStreamWriter out, String attributeName, + String value) throws XMLStreamException { + if (value == null || "".equals(value)) { + return; + } + out.writeAttribute(attributeName, value); + } + + /** + * @param wasCached true if the counting part of the facet generation was + * cached. + */ + public void setCountingCached(boolean wasCached) { + this.countCached = wasCached; + } + +} Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java (revision ) @@ -0,0 +1,246 @@ +package org.apache.lucene.search.exposed; + +import junit.framework.TestCase; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.LuceneTestCase; + +import java.io.File; +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.util.*; + +// TODO: Change this to LuceneTestCase but ensure Flex +public class TestGroupTermProvider extends TestCase { + public static final int DOCCOUNT = 10; + private ExposedHelper helper; + + @Override + public void setUp() throws Exception { + super.setUp(); + + helper = new ExposedHelper(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + helper.close(); + } + + public void testIndexGeneration() throws Exception { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + long termCount = 0; + TermsEnum terms = MultiFields.getFields(reader). + terms(ExposedHelper.ID).iterator(); + while (terms.next() != null) { + assertEquals("The ID-term #" + termCount + " should be correct", + ExposedHelper.ID_FORMAT.format(termCount), + terms.term().utf8ToString()); + termCount++; + } + + assertEquals("There should be the right number of terms", + DOCCOUNT, termCount); + reader.close(); + } + + public void testOrdinalAccess() throws IOException { + String[] fieldNames = new String[]{"a"}; + helper.createIndex( DOCCOUNT, Arrays.asList(fieldNames), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + + ArrayList plainExtraction = new ArrayList(DOCCOUNT); + for (String fieldName: fieldNames) { + // FIXME: The ordinals are not comparable to the exposed ones! + TermsEnum terms = + MultiFields.getFields(reader).terms(fieldName).iterator(); + while (terms.next() != null) { + plainExtraction.add(terms.term().utf8ToString()); + } + } + + TermProvider groupProvider = ExposedFactory.createProvider( + reader, "TestGroup", Arrays.asList(fieldNames), null, false, "null"); + + ArrayList exposedOrdinals = new ArrayList(DOCCOUNT); + for (int i = 0 ; i < groupProvider.getOrdinalTermCount() ; i++) { + exposedOrdinals.add(groupProvider.getTerm(i).utf8ToString()); + } + + assertEquals("The two lists of terms should be of equal length", + plainExtraction.size(), exposedOrdinals.size()); + // We sort as the order of the terms is not significant here + Collections.sort(plainExtraction); + Collections.sort(exposedOrdinals); + for (int i = 0 ; i < plainExtraction.size() ; i++) { + assertEquals("The term at index " + i + " should be correct", + plainExtraction.get(i), exposedOrdinals.get(i)); + } + reader.close(); + + } + public void testTermSortAllDefined() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + testTermSort(reader, Arrays.asList("a")); + reader.close(); + } + + public void testTermSortAllScarce() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + testTermSort(reader, Arrays.asList("even")); + reader.close(); + } + + private void testTermSort(IndexReader index, List fieldNames) + throws IOException { + Collator sorter = Collator.getInstance(new Locale("da")); + + ArrayList plainExtraction = new ArrayList(DOCCOUNT); + // TODO: Make this handle multiple field names + TermsEnum terms = + MultiFields.getFields(index).terms(fieldNames.get(0)).iterator(); + while (terms.next() != null) { + String next = terms.term().utf8ToString(); + plainExtraction.add(next); +// System.out.println("Default order term #" +// + (plainExtraction.size() - 1) + ": " + next); + } + Collections.sort(plainExtraction, sorter); + + TermProvider groupProvider = ExposedFactory.createProvider( + index, "a-group", fieldNames, + ExposedComparators.collatorToBytesRef(sorter), false, "da"); + + ArrayList exposedExtraction = new ArrayList(DOCCOUNT); + Iterator ei = groupProvider.getIterator(false); + int count = 0; + while (ei.hasNext()) { + String next = ei.next().term.utf8ToString(); + exposedExtraction.add(next); +// System.out.println("Exposed sorted term #" + count++ + ": " + next); + } + + assertEquals("The two lists of terms should be of equal length", + plainExtraction.size(), exposedExtraction.size()); + for (int i = 0 ; i < plainExtraction.size() ; i++) { + assertEquals("The term at index " + i + " should be correct", + plainExtraction.get(i), exposedExtraction.get(i)); + } + } + + public void testDocIDMapping() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + Collator sorter = Collator.getInstance(new Locale("da")); + + ArrayList plain = + new ArrayList(DOCCOUNT); + for (int docID = 0 ; docID < reader.maxDoc() ; docID++) { + plain.add(new ExposedHelper.Pair( + docID, reader.document(docID).get("a"), sorter)); +// System.out.println("Plain access added " + plain.get(plain.size()-1)); + } + Collections.sort(plain); + + TermProvider index = + ExposedFactory.createProvider(reader, "docIDGroup", Arrays.asList("a"), + ExposedComparators.collatorToBytesRef(sorter), false, "foo"); + + ArrayList exposed = + new ArrayList(DOCCOUNT); + Iterator ei = index.getIterator(true); + + while (ei.hasNext()) { + final ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + exposed.add(new ExposedHelper.Pair( + doc + tuple.docIDBase, tuple.term.utf8ToString(), sorter)); + } + } + } + Collections.sort(exposed); + + assertEquals("The two docID->term maps should be of equal length", + plain.size(), exposed.size()); + for (int i = 0 ; i < plain.size() ; i++) { +// System.out.println("Sorted docID, term #" + i + ". Plain=" + plain.get(i) +// + ", Exposed=" + exposed.get(i)); + assertEquals("Mapping #" + i + " should be equal", + plain.get(i), exposed.get(i)); + } + reader.close(); + } + + public void testMultiField() throws IOException { + final int DOCCOUNT = 5; + ExposedHelper helper = new ExposedHelper(); + File location = helper.buildMultiFieldIndex(DOCCOUNT); + + IndexReader reader = ExposedIOFactory.getReader(location); + Collator sorter = Collator.getInstance(new Locale("da")); + + TermProvider index = + ExposedFactory.createProvider( + reader, "multiGroup", Arrays.asList("a", "b"), + ExposedComparators.collatorToBytesRef(sorter), false, "foo"); + + ArrayList exposed = + new ArrayList(DOCCOUNT); + Iterator ei = index.getIterator(true); + + long maxID = 0; + while (ei.hasNext()) { + final ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + exposed.add(new ExposedHelper.Pair( + doc + tuple.docIDBase, tuple.term.utf8ToString(), sorter)); + maxID = Math.max(maxID, doc + tuple.docIDBase); + } + } + } + Collections.sort(exposed); + + for (int i = 0 ; i < exposed.size() && i < 10 ; i++) { + System.out.println("Sorted docID, term #" + i + ". Exposed=" + + exposed.get(i)); + } + reader.close(); + assertEquals("The maximum docID in tuples should be equal to the maximum " + + "from the reader", reader.maxDoc()-1, maxID); + } + + public void testTermCount() throws IOException { + helper.createIndex(100, Arrays.asList("a"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + + ExposedRequest.Field fieldRequest = new ExposedRequest.Field( + ExposedHelper.MULTI, null, false, ExposedRequest.LUCENE_ORDER); + ExposedRequest.Group groupRequest = new ExposedRequest.Group("TestGroup", + Arrays.asList(fieldRequest), null, false, ExposedRequest.LUCENE_ORDER); + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, groupRequest); + + assertEquals("The number of unique terms for multi should match", + 25, provider.getUniqueTermCount()); + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/Test.xml =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/Test.xml (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/Test.xml (revision ) @@ -0,0 +1,23 @@ + + + + + Bob Dylan + USA + Columbia + 10.90 + 1985 + + + + Bonnie Tyler + UK + CBS Records + 9.90 + 1988 + + \ No newline at end of file Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestExposedCache.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestExposedCache.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestExposedCache.java (revision ) @@ -0,0 +1,570 @@ +package org.apache.lucene.search.exposed; + +import com.ibm.icu.text.RawCollationKey; +import junit.framework.TestCase; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.File; +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.util.*; + +// TODO: The whole sorting comparison-thingie is deprecated as search-time +// sorting is removed from Lucene trunk in favor of indexing collatorkeys +// TODO: Change this to LuceneTestCase but ensure Flex +public class TestExposedCache extends TestCase { + public static final int DOCCOUNT = 10; + private ExposedHelper helper; + private ExposedCache cache; + + @Override + public void setUp() throws Exception { + super.setUp(); + cache = ExposedCache.getInstance(); +// CodecProvider.setDefaultCodec("Standard"); + helper = new ExposedHelper(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + cache.purgeAllCaches(); + //helper.close(); + } + + public void testPlainSortDump() throws Exception { + helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, new MockAnalyzer( + new Random(), MockTokenizer.WHITESPACE, false)); + Query q = qp.parse("true"); + Sort aSort = new Sort(new SortField("a", SortField.Type.STRING)); + + TopFieldDocs docs = searcher.search(q, null, 100, aSort); + dumpDocs(reader, docs, "a"); + reader.close(); + } + + public void testIndirectIndex() throws IOException { + helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, "foo", Arrays.asList("a"), + ExposedComparators.collatorToBytesRef(Collator.getInstance( + new Locale("da"))), "bar"); + + Iterator iterator = provider.getIterator(false); + while (iterator.hasNext()) { + ExposedTuple tuple = iterator.next(); + assertEquals("The provided term should match ordinal lookup term", + tuple.term.utf8ToString(), + provider.getTerm(tuple.ordinal).utf8ToString()); + assertEquals("The provided term should match indirect lookup term", + tuple.term.utf8ToString(), + provider.getOrderedTerm(tuple.indirect).utf8ToString()); + } + + for (int docID = 0 ; docID < reader.maxDoc() ; docID++) { + String exposed = provider.getOrderedTerm( + provider.getDocToSingleIndirect().get(docID)).utf8ToString(); + String direct = reader.document(docID).get("a"); + assertEquals("Doc #" + docID + " should have the correct a-term", + direct, exposed); + } + } + + public void testIndirectSegment() throws IOException { + helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION). + getSequentialSubReaders()[0]; // SegmentReader + + ExposedRequest.Field fRequest = new ExposedRequest.Field( + "a", ExposedComparators.collatorToBytesRef(Collator.getInstance( + new Locale("da"))), false, "bar"); + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, 0, fRequest, true, true); + + for (int docID = 0 ; docID < reader.maxDoc() ; docID++) { + long indirect = provider.getDocToSingleIndirect().get(docID); + String exposed = indirect == -1 ? null : + provider.getOrderedTerm(indirect).utf8ToString(); + String direct = reader.document(docID).get("a"); + assertEquals("Doc #" + docID + " should have the correct a-term", + direct, exposed); + } + } + + public void testDocCount() throws IOException { + int docs = 12000; + helper.createIndex(docs, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + ExposedCache.getInstance().purgeAllCaches(); + + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, "foo", Arrays.asList("a"), + ExposedComparators.collatorToBytesRef(Collator.getInstance( + new Locale("da"))), "bar"); + assertEquals("The number of documents should be correct", + reader.maxDoc(), provider.getMaxDoc()); + } + + public void testReopen() throws IOException, ParseException { + final int RUNS = 5; + final int UPDATE_SIZE = 5000; + List exposedSortTimes = new ArrayList(RUNS); + + helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 2); + + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer( + new Random(), MockTokenizer.WHITESPACE, false)); + Query query = qp.parse(ExposedHelper.ALL); +// Sort plainSort = new Sort(new SortField("a", new Locale("da"))); + Sort plainSort = new Sort(new SortField("a", SortField.Type.STRING)); + + exposedSortTimes.add(testReopen(reader, query, plainSort)); + for (int i = 1 ; i < RUNS ; i++) { + System.out.println("\nModifying index and re-opening reader " + + (i+1) + "/" + RUNS); + helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 1); + IndexReader oldReader = reader; + reader = reader.reopen(); + assertFalse("The index should change", oldReader == reader); + oldReader.close(); + exposedSortTimes.add(testReopen(reader, query, plainSort)); + } + reader.close(); + long total = 0; + for (long t: exposedSortTimes) { + total += t; + } + System.out.println( + "\n" + RUNS + " exposed sortings with doc#-delta " + UPDATE_SIZE + + " between each took " + total + + " ms split in " + exposedSortTimes); + } + + private long testReopen( + IndexReader reader, Query query, Sort plainSort) throws IOException { + final int docCount = reader.maxDoc(); + final int MAX_HITS = 100; + Sort exposedSort = new Sort(new SortField( + "a", + new ExposedFieldComparatorSource(reader, null))); + //new ExposedFieldComparatorSource(reader, new Locale("da")))); + + IndexSearcher searcher = new IndexSearcher(reader); + long plainTime = System.currentTimeMillis(); + TopFieldDocs plainDocs = searcher.search(query, MAX_HITS, plainSort); + plainTime = System.currentTimeMillis() - plainTime; + + long exposedTime = System.currentTimeMillis(); + // TODO: Check for fillfields + TopFieldDocs exposedDocs = searcher.search(query, MAX_HITS, exposedSort); + exposedTime = System.currentTimeMillis() - exposedTime; + System.out.println( + "Reopen sort test extracted " + + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits in " + + plainTime + " ms for standard Lucene collator sort and " + + exposedTime + " ms for exposed sort"); +// dumpDocs(reader, plainDocs, sortField); +// dumpDocs(reader, exposedDocs, sortField); + assertEquals("The two search results should be equal", + plainDocs, exposedDocs); + return exposedTime; + } + + public void testExposedSortWithNullValues() throws Exception { + testExposedSort(ExposedHelper.EVEN_NULL, DOCCOUNT, false); + } + + public void testExposedSortWithRandom() throws Exception { + testExposedSort("a", DOCCOUNT, false); + } + + public void testExposedSortWithRandomReverse() throws Exception { + testExposedSort("a", DOCCOUNT, false); + helper.close(); + cache.purgeAllCaches(); + helper = new ExposedHelper(); + testExposedSort("a", DOCCOUNT, false, true); + } + + public void testSpeedExposedSortWithRandom() throws Exception { +// testExposedSort("a", 150000, true); Fails at GrowingMutable.get from compareBottom with java.lang.ArrayIndexOutOfBoundsException: 22674 + testExposedSort("a", 20000, true); + } + + private void testExposedSort(String sortField, int docCount, boolean feedback) + throws IOException, ParseException { + testExposedSort(sortField, docCount, feedback, false); + } + + /* + TODO: Create a new test that actually uses Collator-based sorting + */ + private void testExposedSort( + String sortField, int docCount, boolean feedback, boolean reverse) + throws IOException, ParseException { + helper.createIndex(docCount, Arrays.asList("a", "b"), 20, 2); +// testExposedSort(ExposedHelper.INDEX_LOCATION, new Locale("da"), + testExposedSort(ExposedHelper.INDEX_LOCATION, null, + sortField, docCount, feedback, reverse); + } + + public void testExternallyGeneratedIndex() throws Exception { + File external = new File("/home/te/projects/lucene4index"); + if (!external.exists()) { + return; + } + testExposedSort(external, new Locale("da"), "sort_title", 10, true, false); + } + + public void testExposedNaturalOrder() throws IOException, ParseException { + helper.createIndex(DOCCOUNT, Arrays.asList("a"), 20, 2); + testExposedSort(ExposedHelper.INDEX_LOCATION, null, "a", 10, true, false); + } + + // locale can be null + private void testExposedSort(File index, Locale locale, String sortField, + int docCount, boolean feedback, boolean reversed) + throws IOException, ParseException { + if (locale != null) { + throw new UnsupportedOperationException( + "Locale-based sort not supported at this time since it was removed " + + "from Lucene 4 trunk"); + } + final int MAX_HITS = 50; + int retries = feedback ? 5 : 1; + + IndexReader reader = ExposedIOFactory.getReader(index); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer( + new Random(), MockTokenizer.WHITESPACE, false)); + Query q = qp.parse(ExposedHelper.ALL); + Sort aPlainSort = new Sort( + new SortField(sortField, SortField.Type.STRING, reversed)); + // new Sort(new SortField(sortField, locale, reversed)); + Sort aExposedSort = new Sort(new SortField( + sortField, new ExposedFieldComparatorSource(reader, locale), reversed)); + + for (int run = 0 ; run < retries ; run++) { + long plainTime = System.currentTimeMillis(); + TopFieldDocs plainDocs = searcher.search(q, MAX_HITS, aPlainSort); + plainTime = System.currentTimeMillis() - plainTime; + + long exposedTime = System.currentTimeMillis(); + TopFieldDocs exposedDocs = searcher.search(q, MAX_HITS, aExposedSort); + exposedTime = System.currentTimeMillis() - exposedTime; + System.out.println( + "Sorting on field " + sortField + " with " + + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits took " + + plainTime + " ms for standard Lucene collator sort and " + + exposedTime + " ms for exposed sort"); +// dumpDocs(reader, plainDocs, sortField); +// dumpDocs(reader, exposedDocs, sortField); + assertEquals("The two search results should be equal", + plainDocs, exposedDocs); + } + reader.close(); + } + + public void testRawSort() throws IOException { +// final int[] SIZES = new int[]{10, 1000, 20000, 100000}; + final int[] SIZES = new int[]{20000}; + final int RUNS = 3; + Random random = new Random(87); + long createTime = System.currentTimeMillis(); + for (int size: SIZES) { + List terms = new ArrayList(size); + for (int termPos = 0 ; termPos < size ; termPos++) { + String term = ExposedHelper.getRandomString( + random, ExposedHelper.CHARS, 1, 20) + termPos; + terms.add(new BytesRef(term)); + } + System.out.println("Created " + size + " random BytesRefs in " + + (System.currentTimeMillis() - createTime) + "ms"); + + Comparator comparator = ExposedComparators.collatorToBytesRef( + Collator.getInstance(new Locale("da"))); + + for (int run = 0 ; run < RUNS ; run++) { + Collections.shuffle(terms); + + long sortTime = System.currentTimeMillis(); + Collections.sort(terms, comparator); + sortTime = System.currentTimeMillis() - sortTime; + System.out.println("Dumb sorted " + + ExposedUtil.time("bytesrefs", terms.size(), sortTime)); + + long keyTime = System.currentTimeMillis(); + Collator plainCollator = Collator.getInstance(new Locale("da")); + List keys = new ArrayList(size); + for (BytesRef term: terms) { + RawCollationKey key = new RawCollationKey(); + keys.add(plainCollator.getRawCollationKey(term.utf8ToString(), key)); + } + keyTime = System.currentTimeMillis() - keyTime; + System.out.println("Created " + + ExposedUtil.time("CollatorKeys", size, keyTime)); + Collections.shuffle(keys); + + long keySortTime = System.currentTimeMillis(); + Collections.sort(keys); + keySortTime = System.currentTimeMillis() - keySortTime; + System.out.println("Collator sorted " + + ExposedUtil.time("CollatorKeys", size, keySortTime)); + } + System.out.println(""); + } + } + + public void testPlainOrdinalAccess() throws IOException { + final int[] SIZES = new int[]{10, 1000, 50000}; + final int RETRIES = 6; + for (int size: SIZES) { + System.out.println("\nMeasuring for " + size + " documents"); + helper.close(); + helper = new ExposedHelper(); + helper.createIndex(size, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION). + getSequentialSubReaders()[0]; + TermsEnum terms = reader.fields().terms("a").iterator(); + long num = reader.fields().terms("a").getUniqueTermCount(); + + Random random = new Random(); + for (int i = 0 ; i < RETRIES ; i++) { + boolean doRandom = i % 2 == 0; + long firstHalf = System.currentTimeMillis(); + for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) { + try { + terms.seekExact(doRandom ? random.nextInt(ordinal+1) : ordinal); + } catch (UnsupportedOperationException e) { + System.out.println("The current codec for field 'a' does not " + + "support ordinal-based access. Skipping test"); + return; + } + BytesRef term = terms.term(); + assertNotNull(term); // To avoid optimization of terms.term() + } + firstHalf = System.currentTimeMillis() - firstHalf; + + long secondHalf = System.currentTimeMillis(); + for (long ordinal = num / 2 ; ordinal < num ; ordinal++) { + terms.seekExact( + doRandom ? random.nextInt((int) (ordinal+1)) : ordinal); + BytesRef term = terms.term(); + assertNotNull(term); // To avoid optimization of terms.term() + } + secondHalf = System.currentTimeMillis() - secondHalf; + + System.out.println("Seeked " + num + " ordinals " + + (doRandom ? "randomly" : "sequent.") + ". " + + "First half: " + firstHalf + " ms, second half: " + + secondHalf + " ms" + + (firstHalf + secondHalf == 0 ? "" : ". " + + num / (firstHalf + secondHalf) + " seeks/ms")); + } + reader.close(); + } + } + + public void testExposedOrdinalAccess() throws IOException { + final int[] SIZES = new int[]{10, 1000, 50000}; + final int RETRIES = 6; + for (int size: SIZES) { + System.out.println("\nMeasuring for " + size + " documents by exposed"); + helper.close(); + helper = new ExposedHelper(); + helper.createIndex(size, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION). + getSequentialSubReaders()[0]; + ExposedRequest.Field request = + new ExposedRequest.Field("a", null, false, "foo"); + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, 0, request, true, true); + long num = provider.getMaxDoc(); + + Random random = new Random(); + for (int i = 0 ; i < RETRIES ; i++) { + boolean doRandom = i % 2 == 0; + long firstHalf = System.currentTimeMillis(); + for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) { + BytesRef term = provider.getTerm( + doRandom ? random.nextInt(ordinal+1) : ordinal); + assertNotNull(term); // To avoid optimization of terms.term() + } + firstHalf = System.currentTimeMillis() - firstHalf; + + long secondHalf = System.currentTimeMillis(); + for (long ordinal = num / 2 ; ordinal < num ; ordinal++) { + BytesRef term = provider.getTerm( + doRandom ? random.nextInt((int) (ordinal+1)) : ordinal); + assertNotNull(term); // To avoid optimization of terms.term() + } + secondHalf = System.currentTimeMillis() - secondHalf; + + System.out.println("Requested " + (num-1) + " ordinals " + + (doRandom ? "randomly" : "sequent.") + ". " + + "First half: " + firstHalf + " ms, second half: " + + secondHalf + " ms" + + (firstHalf + secondHalf == 0 ? "" : ". " + + num / (firstHalf + secondHalf) + " seeks/ms")); + } + reader.close(); + } + } + /* + +Measuring for 50000 documents +Created 50000 document index with 4 fields with average term length 10 and total size 7MB in 2409ms +org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@38a36b53 +org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@d694eca +Chunk sorted 0-19999: 20000 ordinals in 1061 ms: ~= 18 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/255151, lookups=20000 (66 ms ~= 300 lookups/ms), readAheads=19999 +Chunk sorted 20000-25000: 5001 ordinals in 179 ms: ~= 27 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/54131, lookups=5001 (2 ms ~= 1722 lookups/ms), readAheads=5000 +Chunk merged 2 chunks in 234 ms: ~= 0 chunks/ms aka 25001 terms in 234 ms: ~= 106 terms/ms: CachedTermProvider cacheSize=20000, misses=15/49938, lookups=25804 (32 ms ~= 789 lookups/ms), readAheads=25797 +Chunk total sort for field a: 25001 terms in 1478 ms: ~= 16 terms/ms +Chunk sorted 0-19999: 20000 ordinals in 523 ms: ~= 38 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/257147, lookups=20000 (12 ms ~= 1566 lookups/ms), readAheads=19999 +Chunk sorted 20000-24998: 4999 ordinals in 112 ms: ~= 44 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/53847, lookups=4999 (2 ms ~= 1752 lookups/ms), readAheads=4998 +Chunk merged 2 chunks in 116 ms: ~= 0 chunks/ms aka 24999 terms in 116 ms: ~= 215 terms/ms: CachedTermProvider cacheSize=20000, misses=21/49996, lookups=25217 (15 ms ~= 1674 lookups/ms), readAheads=25210 +Chunk total sort for field a: 24999 terms in 753 ms: ~= 33 terms/ms +FacetGroup total iterator construction: 50000 ordinals in 2246 ms: ~= 22 ordinals/ms +TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=411/25001, lookups=26406 (33 ms ~= 798 lookups/ms), readAheads=26181 +TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=390/24999, lookups=26092 (32 ms ~= 799 lookups/ms), readAheads=25868 +FacetGroup ordinal iterator depletion from 2 providers: 50001 ordinals in 426 ms: ~= 117 ordinals/ms +Got 50001 ordered ordinals in 2673ms: ~18 terms/ms + + */ + public void testTiming() throws IOException { +// final int[] SIZES = new int[]{10, 1000, 20000, 50000}; + final int[] SIZES = new int[]{20000}; + for (int size: SIZES) { + System.out.println("\nMeasuring for " + size + " documents"); + helper.close(); + helper = new ExposedHelper(); + helper.createIndex(size, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + ExposedCache.getInstance().purgeAllCaches(); + + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, "foo", Arrays.asList("a"), + ExposedComparators.collatorToBytesRef(Collator.getInstance( + new Locale("da"))), "bar"); + assertEquals("The number of ordinal accessible terms should match", + size, provider.getOrdinalTermCount()); + + long sortTime = System.currentTimeMillis(); + PackedInts.Reader orderedOrdinals = provider.getOrderedOrdinals(); + sortTime = System.currentTimeMillis() - sortTime; + System.out.println("Got " + orderedOrdinals.size() + + " ordered ordinals in " + sortTime + "ms" + + (sortTime == 0 ? "" : ": ~" + (orderedOrdinals.size() / sortTime) + + " terms/ms")); + reader.close(); +/* Iterator iterator = provider.getIterator(false); + while (iterator.hasNext()) { + ExposedTuple tuple = iterator.next(); + assertEquals("The provided term should match ordinal lookup term", + tuple.term.utf8ToString(), + provider.getTerm(tuple.ordinal).utf8ToString()); + assertEquals("The provided term should match indirect lookup term", + tuple.term.utf8ToString(), + provider.getOrderedTerm(tuple.indirect).utf8ToString()); + } + + for (int docID = 0 ; docID < reader.maxDoc() ; docID++) { + String exposed = provider.getOrderedTerm( + provider.getDocToSingleIndirect().get(docID)).utf8ToString(); + String direct = reader.document(docID).get("a"); + assertEquals("Doc #" + docID + " should have the correct a-term", + direct, exposed); + }*/ + } + } + + public static void assertEquals(String message, TopFieldDocs expected, + TopFieldDocs actual) { + assertEquals(message + ". Expected length " + expected.scoreDocs.length + + " got " + actual.scoreDocs.length, + expected.scoreDocs.length, actual.scoreDocs.length); + for (int i = 0 ; i < actual.scoreDocs.length ; i++) { + ScoreDoc e = expected.scoreDocs[i]; + ScoreDoc a = actual.scoreDocs[i]; + String ef = ((FieldDoc)e).fields[0] == null ? "" + : ((BytesRef)((FieldDoc)e).fields[0]).utf8ToString(); + String af = ((FieldDoc)a).fields[0] == null ? "" // TODO: Is "" == null? + : ((BytesRef)((FieldDoc)a).fields[0]).utf8ToString(); + if (e.doc != a.doc) { + System.out.println("The 10 first expected hits:"); + dumpResult(expected.scoreDocs); + System.out.println("The 10 first actual hits:"); + dumpResult(actual.scoreDocs); + } + assertEquals(String.format( + "%s. The docID for hit#%d/%d should be correct. " + + "Expected %d with term '%s', got %d with term '%s'", + message, i+1, expected.scoreDocs.length, e.doc, ef, a.doc, af), + e.doc, a.doc); + + assertEquals(message + ". The sort value for hit#" + (i+1) + + "/" + expected.scoreDocs.length + " should be " + + ef + " but was " + af, + ef, af); + } + } + + // Max 10 + private static void dumpResult(ScoreDoc[] scoreDocs) { + for (int i = 0 ; i < scoreDocs.length && i < 10 ; i++) { + ScoreDoc e = scoreDocs[i]; + String es = ((BytesRef)((FieldDoc)scoreDocs[i]).fields[0]).utf8ToString(); + System.out.println("Hit #" + i + ", doc=" + e.doc + ", term='" + es + ""); + } + } + + private void dumpDocs(IndexReader reader, TopFieldDocs docs, String field) + throws IOException { + final int MAX = 20; + int num = Math.min(docs.scoreDocs.length, MAX); + System.out.println("Dumping " + num +"/" + docs.scoreDocs.length + " hits"); + int count = 0; + for (ScoreDoc doc: docs.scoreDocs) { + if (count++ > num) { + break; + } + String sortTerm = ((FieldDoc)doc).fields[0] == null ? "null" + : ((BytesRef)((FieldDoc)doc).fields[0]).utf8ToString(); + System.out.println( + doc.doc + " ID=" + reader.document(doc.doc).get(ExposedHelper.ID) + + ", field " + field + "=" + reader.document(doc.doc).get(field) + + ", sort term=" + sortTerm); + } + } + + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTuple.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTuple.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTuple.java (revision ) @@ -0,0 +1,99 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.BytesRef; + +/** + * A representation of Field, Term, ordinal, indirect and potentially docID. + */ +public class ExposedTuple { + public String field; + /** + * The term itself. Note that this is re-used by the iterator, so users of + * the iterator must finish processing the current term before calling next. + */ + public BytesRef term; + /** + * The ordinal for the term, used for direct lookup. + */ + public long ordinal; + /** + * The indirect for the term, used for lookup with + * {@link TermProvider#getOrderedTerm(long)}. + */ + public long indirect; + /** + * An iterator for docIDs. null if docIDs are not requested or no docIDs + * exists for the term. + * If docIDs are requested and if there are multiple docIDs for the term, + * multiple ExposedTuples with the same term and the same ordinal might be + * used. + */ + // TODO: Consider if deletedDocuments should be provided here + public DocsEnum docIDs; + + /** + * The base to add to all docIDs. + */ + public long docIDBase; + + /* + * The order of the term, relative to the other Terms delivered, when using + * an Iterator from {@link #getExposedTuples}. Starts at 0, increments by at + * most 1 for each subsequent ExposedTuple from the iterator. + */ + //public long order; + + public static long instances = 0; + + public ExposedTuple(String field, BytesRef term, + long ordinal, long indirect, + DocsEnum docIDs, long docIDBase) { + this.field = field; + this.term = term; + this.ordinal = ordinal; + this.indirect = indirect; + this.docIDs = docIDs; + this.docIDBase = docIDBase; + instances++; + } + + public ExposedTuple(String field, BytesRef term, + long ordinal, long indirect) { + this.field = field; + this.term = term; + this.ordinal = ordinal; + this.indirect = indirect; + docIDs = null; + docIDBase = -1; + instances++; + } + + public ExposedTuple(ExposedTuple other) { + this.field = other.field; + this.term = other.term; + this.ordinal = other.ordinal; + this.indirect = other.indirect; + this.docIDs = other.docIDs; + this.docIDBase = other.docIDBase; + instances++; + } + + public String toString() { + return "ExposedTuple(" + field + ":" + + (term == null ? "null" : term.utf8ToString()) + ", ord=" + ordinal + + ", indirect=" + indirect + + ", docIDs " + (docIDs == null ? "not " : "") + "present)"; + } + + // Convenience + public void set(String field, BytesRef term, + long ordinal, long indirect, DocsEnum docIDs, long docIDBase) { + this.field = field; + this.term = term; + this.ordinal = ordinal; + this.indirect = indirect; + this.docIDs = docIDs; + this.docIDBase = docIDBase; + } +} Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/performance.txt =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/performance.txt (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/performance.txt (revision ) @@ -0,0 +1,446 @@ +Some results from TestExposedFacets.testScale on a Dell M6500 laptop: +i7 processor @1.7GHz, PC1333 RAM, SSD. + +******************************************************************************** + +Index = /home/te/projects/index1M (1000000 documents) +used heap after loading index and performing a simple search: 5 MB +Maximum possible memory (Runtime.getRuntime().maxMemory()): 88 MB + +First natural order sorted search for "even:true" with 500000 hits: 2473 ms +Subsequent 5 sorted searches average response time: 23 ms +Hit #0 was doc #494892 with field b b_ d 6K102KM Æ494892 +Hit #1 was doc #995058 with field b b_ 0y995058 +Hit #2 was doc #520388 with field b b_ 520388 +Hit #3 was doc #538316 with field b b_ 538316 +Hit #4 was doc #732952 with field b b_ 7 æ 1 732952 + +Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 1:20 minutes +First faceting for even:true: 128 ms +Subsequent 4 faceting calls (count caching disabled) response times: 75 ms + + + + a_ 000J6iWC743074 + a_00 0nWARnepn NC0zeR509161 + a_000wkNk4f wx9 åd879060 + a_001N Ad1kT5Yldo8GFiO464913 + a_001zX4ÅmWtØy3TnvbQnX867119 + + + a_000wkNk4f wx9 åd879060 + a_00201542 + a_00281650 + a_002ylvz 841932 + a_å ååØLcmoJNæ7KrBzX320280 + + + A + G + E + Y + L + + + +Initial lookup pool request (might result in structure building): 0:33 minutes +First index lookup for "even:true": 56 ms +Subsequent 91 index lookups average response times: 0 ms + + + + a_xsRLgqPP1T åRisÆbS626748 + a_xSRR82205 + a_xsRRarvwT3m311451 + a_XsRs704893 + a_ xs ruln SRæEgåw709857 + + + +First natural order sorted search for "multi:A" with 95222 hits: 6 ms +Subsequent 5 sorted searches average response time: 5 ms +Hit #0 was doc #264004 with field b b_ VJbLxZ 264004 +Hit #1 was doc #709779 with field b b_ sYdDc CAe709779 +Hit #2 was doc #403611 with field b b_ 0403611 +Hit #3 was doc #938299 with field b b_ 0Å 2PloF938299 +Hit #4 was doc #714795 with field b b_ 1gHK714795 + +Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms +First faceting for multi:A: 23 ms +Subsequent 4 faceting calls (count caching disabled) response times: 23 ms + + + + a_ 000J6iWC743074 + a_00 0nWARnepn NC0zeR509161 + a_000wkNk4f wx9 åd879060 + a_001N Ad1kT5Yldo8GFiO464913 + a_001zX4ÅmWtØy3TnvbQnX867119 + + + a_002iu283037 + a_009 jot 6959344 + a_00b9KwdoWx 94R185619 + a_ 00BP973393 + a_åÅÅP HCDM159646 + + + A + E + V + D + L + + + +Initial lookup pool request (might result in structure building): 0 ms +First index lookup for "multi:A": 42 ms +Subsequent 91 index lookups average response times: 0 ms + + + + a_xsRLgqPP1T åRisÆbS626748 + a_xSRR82205 + a_xsRRarvwT3m311451 + a_XsRs704893 + a_ xs ruln SRæEgåw709857 + + + + +Used memory with sort, facet and index lookup structures intact: 68 MB +Total test time: 1:57 minutes + +******************************************************************************** + +Index = /home/te/projects/index10M (10000000 documents) +used heap after loading index and performing a simple search: 25 MB +Maximum possible memory (Runtime.getRuntime().maxMemory()): 910 MB + +First natural order sorted search for "even:true" with 5000000 hits: 0:21 minutes +Subsequent 5 sorted searches average response time: 227 ms +Hit #0 was doc #494892 with field b b_ d 6K102KM Æ494892 +Hit #1 was doc #4618566 with field b b_ 43ÅvMN4618566 +Hit #2 was doc #9336724 with field b b_ 6xoc9336724 +Hit #3 was doc #7145374 with field b b_ 7145374 +Hit #4 was doc #8788920 with field b b_ 8788920 + +Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 10:08 minutes +First faceting for even:true: 934 ms +Subsequent 4 faceting calls (count caching disabled) response times: 897 ms + + + + a_0000lcwbWX å 4684894 + a_0002nUXFCÆBWG37fx8464891 + a_0003867599 + a_0003bPt lj7vQE8594334 + a_0006øBK9290537 + + + a_S8252004 + a_T1678792 + a_E1319214 + a_G2886318 + a_åååÅ ø jG8104444 + + + E + S + V + Q + G + + + +Initial lookup pool request (might result in structure building): 3:21 minutes +First index lookup for "even:true": 348 ms +Subsequent 91 index lookups average response times: 2 ms + + + + a_X sNQBjHT5010537 + a_XSNqC20 iååg545448 + a_XsnQDroguT 8387531 + a_xsNQsOMqv6496008 + a_xsNqYj9405590 + + + +First natural order sorted search for "multi:A" with 947218 hits: 99 ms +Subsequent 5 sorted searches average response time: 45 ms +Hit #0 was doc #9748747 with field b b_ JhåDx6IJuT9748747 +Hit #1 was doc #4041747 with field b b_ VHjwP4041747 +Hit #2 was doc #2125062 with field b b_ WSpWmbøs kSY2125062 +Hit #3 was doc #6734767 with field b b_ 0XxYØ 6734767 +Hit #4 was doc #9085678 with field b b_ 1yXUZ0RS2øC k f9085678 + +Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms +First faceting for multi:A: 267 ms +Subsequent 4 faceting calls (count caching disabled) response times: 233 ms + + + + a_0000lcwbWX å 4684894 + a_0002nUXFCÆBWG37fx8464891 + a_0003867599 + a_0003bPt lj7vQE8594334 + a_0006øBK9290537 + + + a_000I 5346125 + a_000wkNk4f wx9 åd879060 + a_000ÆZqiUOÅ59BWx47633391 + a_001019122 + a_åååÆiYyBGzt2jLLXV4931245 + + + A + U + E + V + Y + + + +Initial lookup pool request (might result in structure building): 0 ms +First index lookup for "multi:A": 99 ms +Subsequent 91 index lookups average response times: 2 ms + + + + a_X sNQBjHT5010537 + a_XSNqC20 iååg545448 + a_XsnQDroguT 8387531 + a_xsNQsOMqv6496008 + a_xsNqYj9405590 + + + + +Used memory with sort, facet and index lookup structures intact: 685 MB +Total test time: 14:00 minutes + +******************************************************************************** + +Index = /home/te/projects/index1M (1000000 documents) +used heap after loading index and performing a simple search: 6 MB +Maximum possible memory (Runtime.getRuntime().maxMemory()): 88 MB + +First natural order sorted search for "even:true" with 500000 hits: 2586 ms +Subsequent 5 sorted searches average response time: 23 ms +Hit #0 was doc #494892 with field b b_ d 6K102KM Æ494892 +Hit #1 was doc #995058 with field b b_ 0y995058 +Hit #2 was doc #520388 with field b b_ 520388 +Hit #3 was doc #538316 with field b b_ 538316 +Hit #4 was doc #732952 with field b b_ 7 æ 1 732952 + +Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=index, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0:44 minutes +First faceting for even:true: 120 ms +Subsequent 4 faceting calls (count caching disabled) response times: 66 ms + + + + a_ 24q937966 + a_ 7GY4XFvBjyvef966757 + a_ 0XjÆ 964647 + a_ 4PylI0Aj88S975575 + a_ 7 O826038 + + + a_ 7 O826038 + a_ 984110 + a_ dmo øKåyikbje585780 + a_ LuÅhAøObæ2gy968592 + a_øøå MnKcÆ7670310 + + + A + G + E + Y + L + + + +Initial lookup pool request (might result in structure building): 0:15 minutes +First index lookup for "even:true": 26 ms +Subsequent 91 index lookups average response times: 0 ms + + + + a_ÆÅnyF474489 + a_ÆÅoHx4IxSezWtPØq7110803 + a_ÆÅpQTpFzQnR204607 + a_ÆÅqY 2IF2cI488588 + a_ÆÅqg2OrD202359 + + + +First natural order sorted search for "multi:A" with 95222 hits: 6 ms +Subsequent 5 sorted searches average response time: 4 ms +Hit #0 was doc #264004 with field b b_ VJbLxZ 264004 +Hit #1 was doc #709779 with field b b_ sYdDc CAe709779 +Hit #2 was doc #403611 with field b b_ 0403611 +Hit #3 was doc #938299 with field b b_ 0Å 2PloF938299 +Hit #4 was doc #714795 with field b b_ 1gHK714795 + +Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=index, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms +First faceting for multi:A: 22 ms +Subsequent 4 faceting calls (count caching disabled) response times: 20 ms + + + + a_ 24q937966 + a_ 7GY4XFvBjyvef966757 + a_ 0XjÆ 964647 + a_ 4PylI0Aj88S975575 + a_ 7 O826038 + + + a_ D7HNademsP2Mvws749426 + a_ E3ØæcYx3fD751117 + a_ MRØZ fH747008 + a_ UBxL911779 + a_øø7hh oØn642043 + + + A + E + V + D + L + + + +Initial lookup pool request (might result in structure building): 0 ms +First index lookup for "multi:A": 7 ms +Subsequent 91 index lookups average response times: 0 ms + + + + a_ÆÅnyF474489 + a_ÆÅoHx4IxSezWtPØq7110803 + a_ÆÅpQTpFzQnR204607 + a_ÆÅqY 2IF2cI488588 + a_ÆÅqg2OrD202359 + + + + +Used memory with sort, facet and index lookup structures intact: 66 MB +Total test time: 1:03 minutes + +******************************************************************************** + +Index = /home/te/projects/index10M (10000000 documents) +used heap after loading index and performing a simple search: 25 MB +Maximum possible memory (Runtime.getRuntime().maxMemory()): 910 MB + +First natural order sorted search for "even:true" with 5000000 hits: 0:22 minutes +Subsequent 5 sorted searches average response time: 229 ms +Hit #0 was doc #494892 with field b b_ d 6K102KM Æ494892 +Hit #1 was doc #4618566 with field b b_ 43ÅvMN4618566 +Hit #2 was doc #9336724 with field b b_ 6xoc9336724 +Hit #3 was doc #7145374 with field b b_ 7145374 +Hit #4 was doc #8788920 with field b b_ 8788920 + +Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=index, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 4:46 minutes +First faceting for even:true: 865 ms +Subsequent 4 faceting calls (count caching disabled) response times: 788 ms + + + + a_ 09619514 + a_ 0mXWfE vI 9uPm9544033 + a_ 15w 8sMY d3ææØfD7075330 + a_ 19497740 + a_ 1Mb589885332 + + + a_T1678792 + a_G2886318 + a_E1319214 + a_ 19497740 + a_øøæCq Ep40rYØc96 7860482 + + + E + S + V + Q + G + + + +Initial lookup pool request (might result in structure building): 1:41 minutes +First index lookup for "even:true": 318 ms +Subsequent 91 index lookups average response times: 2 ms + + + + a_8åzåå eBDÆYVg5LÆokKx9224801 + a_8åÅ5HZpj58819757 + a_8åÅ8858061 + a_8åÅmLJJ7W8977610 + a_8åÆ8d XJ QøVsypF0D8905889 + + + +First natural order sorted search for "multi:A" with 947218 hits: 101 ms +Subsequent 5 sorted searches average response time: 51 ms +Hit #0 was doc #9748747 with field b b_ JhåDx6IJuT9748747 +Hit #1 was doc #4041747 with field b b_ VHjwP4041747 +Hit #2 was doc #2125062 with field b b_ WSpWmbøs kSY2125062 +Hit #3 was doc #6734767 with field b b_ 0XxYØ 6734767 +Hit #4 was doc #9085678 with field b b_ 1yXUZ0RS2øC k f9085678 + +Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=index, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms +First faceting for multi:A: 220 ms +Subsequent 4 faceting calls (count caching disabled) response times: 220 ms + + + + a_ 09619514 + a_ 0mXWfE vI 9uPm9544033 + a_ 15w 8sMY d3ææØfD7075330 + a_ 19497740 + a_ 1Mb589885332 + + + a_ 9k4sBvbfBO9554703 + a_ Ce VøjRI3R9532980 + a_ Lq x T 9675445 + a_ 3Å4ØQØxibSgØmÅ9980036 + a_øøø7827179 + + + A + U + E + V + Y + + + +Initial lookup pool request (might result in structure building): 0 ms +First index lookup for "multi:A": 92 ms +Subsequent 91 index lookups average response times: 2 ms + + + + a_8åzåå eBDÆYVg5LÆokKx9224801 + a_8åÅ5HZpj58819757 + a_8åÅ8858061 + a_8åÅmLJJ7W8977610 + a_8åÆ8d XJ QøVsypF0D8905889 + + + + +Used memory with sort, facet and index lookup structures intact: 648 MB +Total test time: 6:58 minutes + +******************************************************************************** +******************************************************************************** +******************************************************************************** Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedSettings.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedSettings.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedSettings.java (revision ) @@ -0,0 +1,61 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.util.packed.PackedInts; + +public class ExposedSettings { + public static enum PRIORITY {memory, speed; + public static PRIORITY fromString(String p) { + if (memory.toString().equals(p)) { + return memory; + } else if (speed.toString().equals(p)) { + return speed; + } + throw new IllegalArgumentException( + "The String '" + p + "' was not recognized as a priority"); + } + } + public static final PRIORITY DEFAULT_PRIORITY = PRIORITY.memory; + + /** + * The priority determines what is most important: memory or speed. + * The setting is used whenever a PackedInts-structure is created to determine + * if rounding to byte, short, int or long representations should be used or + * if the optimal space saving representation should be used (no rounding). + *

+ * The speed gain depends very much on the actual index and the setup for + * sorting, faceting and index lookup, but a rough rule of thumb is that + * memory usage increases 25-50% and speed increases 25-50% for faceting and + * initial index look ups when speed is chosen. + */ + public static PRIORITY priority = DEFAULT_PRIORITY; + + /** + * If true, debug messages will be written to stdout. + */ + public static boolean debug = false; + + /** + * Construct a PackedInts.Mutable based on the given values and the overall + * priority between memory usage and speed. If the maxValue is below 2^4 of + * between 2^32 and 2^40, a standard PackedInts.Mutable is created, else the + * maxValue is rounded up to 2^8-1, 2^16-1, 2^32-1 or 2^64-1; + * @param valueCount the number of values that the mutator should hold. + * @param maxValue the maximum value that will ever be inserted. + * @return a mutable optimized for either speed or memory. + */ + public static PackedInts.Mutable getMutable(int valueCount, long maxValue) { + int bitsRequired = PackedInts.bitsRequired(maxValue); + switch (priority) { + case memory: return PackedInts.getMutable(valueCount, bitsRequired); + case speed: if (bitsRequired <= 4 || + (bitsRequired > 32 && bitsRequired < 40)) { // TODO: Consider the 40 + return PackedInts.getMutable(valueCount, bitsRequired); + } else { + return PackedInts.getMutable( + valueCount, PackedInts.getNextFixedSize(bitsRequired)); + } + default: throw new IllegalArgumentException( + "The priority " + priority + " is unknown"); + } + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java (revision ) @@ -0,0 +1,49 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; + +public final class MockFixedLengthPayloadFilter extends TokenFilter { + private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); + private final Random random; + private final byte[] bytes; + private final Payload payload; + + public MockFixedLengthPayloadFilter(Random random, TokenStream in, int length) { + super(in); + this.random = random; + this.bytes = new byte[length]; + this.payload = new Payload(bytes); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + random.nextBytes(bytes); + payloadAtt.setPayload(payload); + return true; + } else { + return false; + } + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/README.TXT =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/README.TXT (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/README.TXT (revision ) @@ -0,0 +1,11 @@ +The while poc-folder is just for testing. The Mock-classes are copied from the +test-folder in order to be accessible in the Lucene JAR. + + +Get command line usage by calling +java -cp build/lucene-core-4.0-dev.jar org.apache.lucene.search.exposed.poc.ExposedPOC + +Sample command line + + +Note: This patch only supports Lucene flex indexes Index: solr/common-build.xml =================================================================== --- solr/common-build.xml (revision 1145016) +++ solr/common-build.xml (revision ) @@ -93,6 +93,7 @@ property="queryparser.uptodate" classpath.property="queryparser.jar"/> + @@ -137,6 +138,11 @@ + + + + + @@ -164,6 +170,7 @@ + @@ -252,7 +259,7 @@ @@ -271,6 +278,7 @@ + Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockAnalyzer.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockAnalyzer.java (revision ) @@ -0,0 +1,169 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +// Copied from the test-framework +import org.apache.lucene.util.automaton.CharacterRunAutomaton; + +/** + * Analyzer for testing + *

+ * This analyzer is a replacement for Whitespace/Simple/KeywordAnalyzers + * for unit tests. If you are testing a custom component such as a queryparser + * or analyzer-wrapper that consumes analysis streams, its a great idea to test + * it with this analyzer instead. MockAnalyzer has the following behavior: + *

+ * @see MockTokenizer + */ +public final class MockAnalyzer extends Analyzer { + private final CharacterRunAutomaton runAutomaton; + private final boolean lowerCase; + private final CharacterRunAutomaton filter; + private final boolean enablePositionIncrements; + private int positionIncrementGap; + private final Random random; + private Map previousMappings = new HashMap(); + private boolean enableChecks = true; + + /** + * Creates a new MockAnalyzer. + * + * @param random Random for payloads behavior + * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+) + * @param lowerCase true if the tokenizer should lowercase terms + * @param filter DFA describing how terms should be filtered (set of stopwords, etc) + * @param enablePositionIncrements true if position increments should reflect filtered terms. + */ + public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) { + this.random = random; + this.runAutomaton = runAutomaton; + this.lowerCase = lowerCase; + this.filter = filter; + this.enablePositionIncrements = enablePositionIncrements; + } + + /** + * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) + * MockAnalyzer(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false}). + */ + public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase) { + this(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false); + } + + /** + * Create a Whitespace-lowercasing analyzer with no stopwords removal. + *

+ * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) + * MockAnalyzer(random, MockTokenizer.WHITESPACE, true, MockTokenFilter.EMPTY_STOPSET, false}). + */ + public MockAnalyzer(Random random) { + this(random, MockTokenizer.WHITESPACE, true); + } + + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + tokenizer.setEnableChecks(enableChecks); + TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements); + filt = maybePayload(filt, fieldName); + return filt; + } + + private class SavedStreams { + MockTokenizer tokenizer; + TokenFilter filter; + } + + @Override + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + @SuppressWarnings("unchecked") Map map = (Map) getPreviousTokenStream(); + if (map == null) { + map = new HashMap(); + setPreviousTokenStream(map); + } + + SavedStreams saved = map.get(fieldName); + if (saved == null) { + saved = new SavedStreams(); + saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + saved.tokenizer.setEnableChecks(enableChecks); + saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements); + saved.filter = maybePayload(saved.filter, fieldName); + map.put(fieldName, saved); + return saved.filter; + } else { + saved.tokenizer.reset(reader); + return saved.filter; + } + } + + private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) { + Integer val = previousMappings.get(fieldName); + if (val == null) { + val = -1; // no payloads +/* if (LuceneTestCase.rarely(random)) { + switch(random.nextInt(3)) { + case 0: val = -1; // no payloads + break; + case 1: val = Integer.MAX_VALUE; // variable length payload + break; + case 2: val = random.nextInt(12); // fixed length payload + break; + } + }*/ + previousMappings.put(fieldName, val); // save it so we are consistent for this field + } + + if (val == -1) + return stream; + else if (val == Integer.MAX_VALUE) + return new MockVariableLengthPayloadFilter(random, stream); + else + return new MockFixedLengthPayloadFilter(random, stream, val); + } + + public void setPositionIncrementGap(int positionIncrementGap){ + this.positionIncrementGap = positionIncrementGap; + } + + @Override + public int getPositionIncrementGap(String fieldName){ + return positionIncrementGap; + } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } +} Index: solr/contrib/exposed/build.xml =================================================================== --- solr/contrib/exposed/build.xml (revision ) +++ solr/contrib/exposed/build.xml (revision ) @@ -0,0 +1,50 @@ + + + + + + + + + + Alternative facet implementation providing slow startup, low-memory, high performance hierarchical faceting with multiple paths per document. + + + + + + + + + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProvider.java (revision ) @@ -0,0 +1,168 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Order-oriented variation on {@link org.apache.lucene.index.Terms}. + *

+ * Access is primarily through ordinals. This can be done directly with + * {@link #getTerm} and {@link #getField} or indirectly through + * {@link #getOrderedTerm} and {@link #getOrderedField}. Using the ordered + * methods is equivalent to {@code getTerm(getOrder().get(orderPosition))}. + */ +public interface TermProvider { + + /** + * Performs a search for the given term, returning the indirect for the + * term or the nearest indirect if not present. + *

+ * Note: While a binary search is used, this is still somewhat expensive as + * the terms themselves needs to be retrieved. + * @param key the term to search for. + * @return the indirect for the best matching term. + * @throws java.io.IOException if the terms could not be accessed. + */ + public int getNearestTermIndirect(BytesRef key) throws IOException; + + /** + * Performs a search for the given term, returning the indirect for the + * term or the nearest indirect if not present. + *

+ * Note: While a binary search is used, this is still somewhat expensive as + * the terms themselves needs to be retrieved. + * @param key the term to search for. + * @param startTermPos where to search from (inclusive). + * @param endTermPos where to search to (exclusive). + * @return the indirect for the best matching term. + * @throws java.io.IOException if the terms could not be accessed. + */ + public int getNearestTermIndirect( + BytesRef key, int startTermPos, int endTermPos) throws IOException; + + /** + * @return a comparator used for sorting. If this is null, natural BytesRef + * order is to be used. + */ + Comparator getComparator(); + + /** + * @return an ID identifying the comparator. If natural BytesRef order is to + * be used, the ID should be {@link ExposedRequest#LUCENE_ORDER}. + */ + String getComparatorID(); + + /** + * @return a short name or description of this provider. + */ + String getDesignation(); // Debugging and feedback + + String getField(long ordinal) throws IOException; + + BytesRef getTerm(long ordinal) throws IOException; + + String getOrderedField(long indirect) throws IOException; + + // Note: indirect might be -1 which should return null + BytesRef getOrderedTerm(long indirect) throws IOException; + + long getUniqueTermCount() throws IOException; + + long getOrdinalTermCount() throws IOException; + + long getMaxDoc(); + + IndexReader getReader(); + + /** + * @return the base to add to every returned docID in order to get the + * index-wide docID. + */ + int getDocIDBase(); + + void setDocIDBase(int base); + + int getReaderHash(); + + /** + * If the provider is a leaf, the hash-value is that of the IndexReader. + * If the provider is a node, the value is the sum of all contained + * term-providers hash-values. + *

+ * This is useful for mapping segment-level docIDs to index-level then a + * group consists of more than one segment. + * @return a hash-value useful for determining reader-equivalence. + */ + int getRecursiveHash(); + + /** + * The ordinals sorted by a previously provided Comparator (typically a + * Collator). + * @return ordinals ordered by Comparator.. + * @throws IOException if the ordinals could not be retrieved. + */ + PackedInts.Reader getOrderedOrdinals() throws IOException; + + /** + * Mapping from docID to the ordered ordinals returned by + * {@link #getOrderedOrdinals()}. Usable for sorting by field value as + * comparisons of documents can be done with {@code + PackedInts.Reader doc2indirect = getDocToSingleIndirect(); + ... + if (doc2indirect.get(docID1) < doc2indirect.get(docID2)) { + System.out.println("Doc #" + docID1 + " comes before doc # " + docID2); + }}. If the term for a document is needed it can be retrieved with + * {@code getOrderedTerm(doc2indirect.get(docID))}. + *

+ * Documents without a corresponding term has the indirect value {@code -1}. + * @return a map from document IDs to the indirect values for the terms for + * the documents. + * @throws IOException if the index could not be accessed, + */ + // TODO: Handle missing value (-1? Max+1? Defined by "empty first"?) + PackedInts.Reader getDocToSingleIndirect() throws IOException; + + /** + * @param collectDocIDs if true, the document IDs are returned as part of the + * tuples. Note that this normally increases the processing time + * significantly. + * @return an iterator over tuples in the order defined for the TermProvider. + * @throws IOException if the iterator could not be constructed. + */ + Iterator getIterator(boolean collectDocIDs) throws IOException; + + DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException; + + /** + * Release all cached values if the provider satisfies the given constraints. + * @param level the level in a provider-tree. This starts at 0 and should + * be incremented by 1 if the provider delegates cache release + * to sub-providers. + * @param keepRoot if true, caches at the root level (level == 0) should not + * be released. + */ + void transitiveReleaseCaches(int level, boolean keepRoot); +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java (revision ) @@ -0,0 +1,429 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.exposed.ExposedSettings; +import org.apache.lucene.search.exposed.ExposedTuple; +import org.apache.lucene.search.exposed.ExposedUtil; +import org.apache.lucene.search.exposed.TermProvider; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.AbstractMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Provides a map from docID to tagIDs (0 or more / document), where tagID + * refers to a tag one out of multiple TermProviders. The mapping is optimized + * towards low memory footprint and extraction of all tags for each given + * document ID. + *

+ * Technical notes: This is essentially a two-dimensional array of integers. + * Dimension 1 is document ID, dimension 2 is references to indirects. + * However, two-dimensional arrays take up a lot of memory so we handle this + * by letting the array doc2ref contain one pointer for each docID into a second + * array refs, which contains the references. We know the number of refs to + * extract for a given docID by looking at the starting point for docID+1. + *

+ * The distribution of the entries in refs is defined by the index layout and + * is assumed to be random. + * TODO: Consider sorting each sub-part of refs and packing it + * The distribution of entries in doc2ref is monotonically increasing. We + * exploit this by having a third array refBase which contains the starting + * points in refs for every 256th entry in doc2ref. This allows us to use less + * bits in doc2ref to represent the pointers into refs. The code for the + * starting point in refs for a given docID is thus + * {@code refBase[docID >>> 8] + doc2ref[docID]}. + */ +public class FacetMap { + // TODO: Consider auto-tuning this value + private static final int BASE_BITS = 8; + private static final int EVERY = (int)Math.pow(2, BASE_BITS); + + private final List providers; + private final int[] indirectStarts; + + private final int[] refBase; + private final PackedInts.Mutable doc2ref; + private final PackedInts.Mutable refs; + + public FacetMap(int docCount, List providers) + throws IOException { + this.providers = providers; + indirectStarts = new int[providers.size() +1]; + int start = 0; + long uniqueTime = -System.currentTimeMillis(); + for (int i = 0 ; i < providers.size() ; i++) { + indirectStarts[i] = start; + start += providers.get(i).getUniqueTermCount(); + } + uniqueTime += System.currentTimeMillis(); + indirectStarts[indirectStarts.length-1] = start; + + refBase = new int[(docCount >>> BASE_BITS) + 1]; +// doc2ref = PackedInts.getMutable(docCount+1, PackedInts.bitsRequired(start)); + long tagExtractTime = - System.currentTimeMillis(); + Map.Entry pair = + extractTags(docCount); + tagExtractTime += System.currentTimeMillis(); + doc2ref = pair.getKey(); + refs = pair.getValue(); + +// System.out.println("Unique count: " + uniqueTime +// + "ms, tag time: " + tagExtractTime + "ms"); + } + + public int getTagCount() { + return indirectStarts[indirectStarts.length-1]; + } + + /* + In order to efficiently populate the ref-structure, we perform a three-pass + run. + Pass 1: The number of references is counted for each document. + Pass 2: The doc2ref and refs-arrays are initialized so that doc2ref points + to the final offsets in refs and so that the first entry in each refs-chunk + designate the offset in the chunk for the next reference to store. + Pass 3: The refs-array is filled by iterating the ExposedTuples for each + TermProvider, adjusting the docID accordingly and storing the tagID in the + refs-array at the position given by doc2ref plus the offset from pass 2. + If the offset was larger than 0, it it decreased. + */ + private Map.Entry extractTags( + int docCount) throws IOException { + if (ExposedSettings.debug) { + System.out.println("Creating facet map for " + providers.size() + + " group" + (providers.size() == 1 ? "" : "s")); + } + // We start by counting the references as this spares us a lot of array + // content re-allocation + final int[] tagCounts = new int[docCount]; // One counter for each doc + long maxRefBlockSize = 0; // from refsBase in blocks of EVERY size + long totalRefs = 0; + + // Fill the tagCounts with the number of tags (references really) for each + // document. + countTags(tagCounts); + + { // Update totalRefs and refBase + int next = 0; + for (int i = 0 ; i < tagCounts.length ; i++) { + if (i == next) { + refBase[i >>> BASE_BITS] = (int)totalRefs; + next += EVERY; + maxRefBlockSize = Math.max(maxRefBlockSize, + totalRefs - (i == 0 ? 0 : refBase[(i-1) >>> BASE_BITS])); + + } + totalRefs += tagCounts[i]; + } + maxRefBlockSize = Math.max(maxRefBlockSize, + totalRefs - (tagCounts.length-1 < EVERY ? 0 : + refBase[(tagCounts.length-2) >>> BASE_BITS])); + if (totalRefs > Integer.MAX_VALUE) { + throw new IllegalStateException( + "The current implementations does not support more that " + + "Integer.MAX_VALUE references to tags. The number of " + + "references was " + totalRefs); + } + } + + final PackedInts.Mutable doc2ref = + ExposedSettings.getMutable(docCount+1, (int)maxRefBlockSize); + + // With the tag counts and the refBase in place, it is possible to fill the + // doc2ref with the correct pointers into the (still non-existing) refs. + initDoc2ref(tagCounts, doc2ref); +/* System.out.print("doc2ref:"); + for (int i = 0 ; i < doc2ref.size() ; i++) { + System.out.print(" " + doc2ref.get(i)); + } + System.out.println(""); + */ + // As we know the number of references we can create the refs-array. + final PackedInts.Mutable refs = ExposedSettings.getMutable( + (int)totalRefs, getTagCount()); + + // We could save a lot of memory by discarding the tagCounts at this point + // and use doc2ref to keep track of pointers. However, this adds some time + // to the overall processing (about 1/3), so we choose speed over ram + + // We are now ready to fill in the actual tagIDs. There will be a lot of + // random writes to the refs-array as we're essentially inverting index + // and value from the TermDocs. + fillRefs(tagCounts, totalRefs, refs); + + // Finally we reduce the doc2ref representation by defining the content of + // refBase and creating a new doc2ref + // Find max +/* { + long reduceTime = -System.currentTimeMillis(); + long max = 0; + for (int i = 0 ; i < doc2ref.size() ; i += EVERY) { + final long count = doc2ref.get( + Math.min(i + EVERY, doc2ref.size()-1)) - doc2ref.get(i); + max = Math.max(max, count); + } + + // Allocate new doc2ref + PackedInts.Mutable reduced = + ExposedSettings.getMutable(doc2ref.size(), max); + + // Adjust bases and doc2refs + for (int i = 0 ; i < doc2ref.size() ; i += EVERY) { + final long base = doc2ref.get(i); + if (refBase[i >>> BASE_BITS] != (int)base) { + System.out.println("Invalid! " + refBase[i >>> BASE_BITS] + " vs " + base); + } + refBase[i >>> BASE_BITS] = (int)base; + + final int to = Math.min(doc2ref.size(), i + EVERY); + for (int docID = i ; docID < to ; docID++) { + reduced.set(docID, doc2ref.get(docID) - base); + } + } + reduceTime += System.currentTimeMillis(); + System.out.println("Reduced doc2ref with " + doc2ref.size() + + " entries and " + doc2ref.getBitsPerValue() + " bits/value from " + + packedSize(doc2ref) + " to " + + + reduced.getBitsPerValue() + " bits/value = " + packedSize(reduced) + + " plus " + refBase.length*4/1024 + " KB for refBase in " + + reduceTime / 1000 + " seconds"); + doc2ref = reduced; + }*/ + return new AbstractMap.SimpleEntry( + doc2ref, refs); + } + + private void fillRefs(final int[] tagCounts, final long totalRefs, + final PackedInts.Mutable refs) throws IOException { + long fillTime = -System.currentTimeMillis(); + for (int providerNum = 0 ; providerNum < providers.size() ; providerNum++) { + final TermProvider provider = providers.get(providerNum); + final long termOffset = indirectStarts[providerNum]; + final Iterator tuples = provider.getIterator(true); + while (tuples.hasNext()) { + final ExposedTuple tuple = tuples.next(); + final long indirect = tuple.indirect + termOffset; + if (tuple.docIDs == null) { + continue; // It happens sometimes with non-expunged deletions + } + + int read; + final DocsEnum.BulkReadResult bulk = tuple.docIDs.getBulkResult(); + final IntsRef docs = bulk.docs; + final int base = (int)tuple.docIDBase; + while ((read = tuple.docIDs.read()) > 0) { + final int to = read + docs.offset; + for (int i = docs.offset ; i < to ; i++) { +// final int docID = ; + // final int refsOrigo = (int)doc2ref.get(docID); +// final int chunkOffset = (int)refs.get(refsOrigo); +// final int chunkOffset = --tagCounts[docID]; + final int refsPos = tagCounts[docs.ints[i] + base]++; + try { + refs.set(refsPos, indirect); + } catch (ArrayIndexOutOfBoundsException e) { + throw new RuntimeException( + "Array index out of bounds. refs.size=" + refs.size() + + ", refs.bitsPerValue=" + refs.getBitsPerValue() + + ", refsPos=" + + refsPos + + ", tuple.indirect+termOffset=" + + tuple.indirect + "+" + termOffset + "=" + + (tuple.indirect+termOffset), e); + } + } + } + +/* + + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + final int refsOrigo = (int)doc2ref.get((int)(doc + tuple.docIDBase)); +// final int chunkOffset = (int)refs.get(refsOrigo); + final int chunkOffset = --tagCounts[((int) (doc + tuple.docIDBase))]; + try { + refs.set(refsOrigo + chunkOffset, tuple.indirect + termOffset); + } catch (ArrayIndexOutOfBoundsException e) { + throw new RuntimeException( + "Array index out of bounds. refs.size=" + refs.size() + + ", refs.bitsPerValue=" + refs.getBitsPerValue() + + ", refsOrigo+chunkOffset=" + + refsOrigo + "+" + chunkOffset + + "=" + (refsOrigo+chunkOffset) + + ", tuple.indirect+termOffset=" + + tuple.indirect + "+" + termOffset + "=" + + (tuple.indirect+termOffset), e); + } +// if (chunkOffset != 0) { +// refs.set(refsOrigo, chunkOffset-1); +// } + } + */ + } + } + fillTime += System.currentTimeMillis(); + if (ExposedSettings.debug) { + System.out.println("Filled map with " + + ExposedUtil.time("references", totalRefs, fillTime)); + } + } + + private void initDoc2ref(int[] tagCounts, PackedInts.Mutable doc2ref) { + long initTime = -System.currentTimeMillis(); + int offset = 0; + for (int i = 0 ; i < tagCounts.length ; i++) { + doc2ref.set(i, offset - refBase[i >>> BASE_BITS]); +// if (tagCounts[i] != 0) { +// refs.set(offset, tagCounts[i]-1); + final int oldOffset = offset; + offset += tagCounts[i]; + tagCounts[i] = oldOffset; + + // } + } + doc2ref.set( + tagCounts.length, offset - refBase[tagCounts.length >>> BASE_BITS]); +// doc2ref.set(doc2ref.size()-1, offset); + initTime += System.currentTimeMillis(); +// System.out.println("Initialized map for " + totalRefs + " references in " +// + initTime / 1000 + " seconds"); + } + + private void countTags(final int[] tagCounts) throws IOException { + long tagCountTime = -System.currentTimeMillis(); + long tupleCount = 0; + long tupleTime = 0; + for (TermProvider provider: providers) { + final Iterator tuples = provider.getIterator(true); + while (tuples.hasNext()) { + tupleTime -= System.nanoTime(); + final ExposedTuple tuple = tuples.next(); + tupleTime += System.nanoTime(); + tupleCount++; + if (tuple.docIDs == null) { + continue; + } +/* int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + tagCounts[(int)(doc + tuple.docIDBase)]++; + }*/ + int read; + DocsEnum.BulkReadResult bulk = tuple.docIDs.getBulkResult(); + final IntsRef intsRef = bulk.docs; + final int base = (int)tuple.docIDBase; + while ((read = tuple.docIDs.read()) > 0) { + final int to = read + intsRef.offset; + for (int i = intsRef.offset ; i < to ; i++) { + tagCounts[base + intsRef.ints[i]]++; + } + } + } + } + tagCountTime += System.currentTimeMillis(); + if (ExposedSettings.debug) { + System.out.println("Counted tag references for " + + ExposedUtil.time("documents", tagCounts.length, tagCountTime) + + ". Retrieved " + + ExposedUtil.time("tuples", tupleCount, tupleTime / 1000000)); + } + } + + /** + * Takes an array where each entry corresponds to a tagID in this facet map + * and increments the counts for the tagIDs associated with the given docID. + * @param tagCounts a structure for counting occurences of tagIDs. + * @param docID an absolute document ID from which to extract tagIDs. + */ + // TODO: Check if static helps speed in this inner loop method + public final void updateCounter(final int[] tagCounts, final int docID) { + final int start = + (int)(refBase[docID >>> BASE_BITS] + doc2ref.get(docID)); + final int end = + (int)(refBase[(docID+1) >>> BASE_BITS] + doc2ref.get(docID+1)); + for (int refI = start ; refI < end ; refI++) { + try { + tagCounts[(int)refs.get(refI)]++; + } catch (Exception ex) { + System.out.println(ex); + } + } + } + + public int[] getIndirectStarts() { + return indirectStarts; + } + + public List getProviders() { + return providers; + } + + public BytesRef getOrderedTerm(final int termIndirect) throws IOException { + for (int i = 0 ; i < providers.size() ; i++) { + if (termIndirect < indirectStarts[i+1]) { + return providers.get(i).getOrderedTerm( + termIndirect- indirectStarts[i]); + } + } + throw new ArrayIndexOutOfBoundsException( + "The indirect " + termIndirect + " was too high. The maximum indirect " + + "supported by the current map is " + + indirectStarts[indirectStarts.length-1]); + } + + /** + * Generates an array of terms for the given docID. This method is normally + * used for debugging and other inspection purposed. + * @param docID the docID from which to request terms. + * @return the terms for a given docID. + * @throws java.io.IOException if the terms could not be accessed. + */ + public BytesRef[] getTermsForDocID(int docID) throws IOException { + final int start = + (int)(refBase[docID >>> BASE_BITS] + doc2ref.get(docID)); + final int end = + (int)(refBase[(docID+1) >>> BASE_BITS] + doc2ref.get(docID+1)); +// System.out.println("Doc " + docID + ", " + start + " -> " + end); + BytesRef[] result = new BytesRef[end - start]; + for (int refI = start ; refI < end ; refI++) { + result[refI - start] = getOrderedTerm((int)refs.get(refI)); + } + return result; + } + + public String toString() { + StringWriter sw = new StringWriter(); + sw.append("FacetMap(#docs=").append(Integer.toString(doc2ref.size()-1)); + sw.append(" (").append(packedSize(doc2ref)).append(")"); + sw.append(", #refs=").append(Integer.toString(refs.size())); + sw.append(" (").append(packedSize(refs)).append(")"); + sw.append(", providers("); + for (int i = 0 ; i < providers.size() ; i++) { + if (i != 0) { + sw.append(", "); + } + sw.append(providers.get(i).toString()); + } + sw.append("))"); + return sw.toString(); + } + + private String packedSize(PackedInts.Reader packed) { + long bytes = (long)packed.size() * packed.getBitsPerValue() / 8 + + refBase.length * 4; + if (bytes > 1048576) { + return bytes / 1048576 + " MB"; + } + if (bytes > 1024) { + return bytes / 1024 + " KB"; + } + return bytes + " bytes"; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagSumIterator.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagSumIterator.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagSumIterator.java (revision ) @@ -0,0 +1,172 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.search.exposed.facet.request.SubtagsConstraints; + +import java.io.IOException; + +/** + * Iterates tags at a given level and sums counts from sub-tags that satisfies + * the constraints from the constraints. + *

+ * next() must be called at least once before any of the getters will return + * meaningful information. + */ +public class TagSumIterator { + final Pool pool; + + final HierarchicalTermProvider provider; + final int delta; + final int[] tagCounts; + + SubtagsConstraints constraints; + int rangeStartPos = 0; // Inclusive + int rangeEndPos = 0; // Exclusive + int level = 0; + + int count = 0; // At the current level + int totalCount = 0; // At all levels + int tagStartPos = 0; // Inclusive + int tagPos = 0; + int tagEndPos = 0; // Exclusive + boolean tagAvailable = false; + + // Ready to use after this + public TagSumIterator(final HierarchicalTermProvider provider, + final SubtagsConstraints constraints, + final int[] tagCounts, + final int rangeStartPos, final int rangeEndPos, + final int level, + final int delta) { + this.provider = provider; + this.tagCounts = tagCounts; + this.delta = delta; + pool = new Pool() { + @Override + public TagSumIterator createElement() { + return new TagSumIterator(this, provider, tagCounts, delta); + } + }; + reset(constraints, rangeStartPos, rangeEndPos, level); + } + + // Remember to call reset after this one + private TagSumIterator(final Pool pool, + final HierarchicalTermProvider provider, + final int[] tagCounts, final int delta) { + this.pool = pool; + this.provider = provider; + this.tagCounts = tagCounts; + this.delta = delta; + } + + public void reset(SubtagsConstraints constraints, + int rangeStartPos, int rangeEndPos, int level) { + this.constraints = constraints; + this.rangeStartPos = rangeStartPos; + this.rangeEndPos = rangeEndPos; + this.level = level; + + count = 0; + totalCount = 0; + tagStartPos = -1; + tagPos = rangeStartPos; + tagEndPos = -1; + tagAvailable = false; + } + + /* + +L P T C +1 0 A (1) (no previous tag) +3 1 A/B/C (2) (The previous tag "A" matches only the first level of "A/B/C") +3 2 A/B/J (1) (The previous tag "A/B/C" matches first and second level of "A/B/J") +2 0 D/E (1) (no previous match) +3 2 D/E/F (1) (The previous tag "D/E" matches both the first and second level of "D/E/F") +3 0 G/H/I (1) (no previous tag) + + */ + + /** + * Skips to the next tag which satisfies the constraints constraints. + * Behaviour is undefined if next() is called after it has returned false. + * @return true if a valid tag is found. + */ + public boolean next() { + tagAvailable = false; + tagEndPos = rangeEndPos; + final int minCount = constraints.getMinCount(); + while (!tagAvailable && tagPos < rangeEndPos) { +// System.out.println(dumpTag(tagStartPos)); + tagEndPos = rangeEndPos; + + while (tagStartPos == -1 && + provider.getLevel(tagPos) < level && tagPos < rangeEndPos) { + tagPos++; + } + + // Check for termination + if ((tagPos > rangeStartPos || provider.getLevel(tagPos) < level) && + !provider.matchLevel(tagPos + delta, level)) { + tagEndPos = tagPos; + return false; + } + + tagStartPos = tagPos; // The start of the potential tag + count = tagCounts[tagStartPos]; + totalCount = count; + tagPos++; // Advance to next + + // If children, sum them to get totalCount for the tag and advance tagPos + while (tagPos < rangeEndPos && + provider.getLevel(tagPos) > level && + provider.getPreviousMatchingLevel(tagPos) >= level) { +// System.out.println("Subsumming for " + tagPos); + TagSumIterator subIterator = pool.acquire(); + subIterator.reset( + constraints.getDeeperLevel(), tagPos, rangeEndPos, level+1); + int subTotalCount = subIterator.countValid(); + tagPos = subIterator.tagEndPos; // Skip processed + totalCount += subTotalCount; + } + + // If totalcount is okay, stop iterating, else start over + if (tagCounts[tagStartPos] < minCount || + totalCount < constraints.getMinTotalCount()) { + continue; + } + tagEndPos = tagPos; + tagAvailable = true; + } + return tagAvailable; + } + + public String getTagInfo() { + try { + return "indirect=" + tagStartPos + + ", level=" + provider.getLevel(tagStartPos) + + ", pLevel=" + provider.getPreviousMatchingLevel(tagStartPos) + + ", tag='" + provider.getOrderedTerm(tagStartPos).utf8ToString() + + "'"; + } catch (IOException e) { + throw new IllegalArgumentException( + "Could not locate the tag for indirect " + tagStartPos, e); + } + } + + private int countValid() { + int total = 0; + while (next()) { + total += getTotalCount(); + } + return total; + } + + /* Getters */ + + public int getCount() { + return count; + } + public int getTotalCount() { + return totalCount; + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenFilter.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenFilter.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenFilter.java (revision ) @@ -0,0 +1,108 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty; +import static org.apache.lucene.util.automaton.BasicAutomata.makeString; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.automaton.BasicOperations; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; + +/** + * A tokenfilter for testing that removes terms accepted by a DFA. + *

+ */ +public final class MockTokenFilter extends TokenFilter { + /** Empty set of stopwords */ + public static final CharacterRunAutomaton EMPTY_STOPSET = + new CharacterRunAutomaton(makeEmpty()); + + /** Set of common english stopwords */ + public static final CharacterRunAutomaton ENGLISH_STOPSET = + new CharacterRunAutomaton(BasicOperations.union(Arrays.asList( + makeString("a"), makeString("an"), makeString("and"), makeString("are"), + makeString("as"), makeString("at"), makeString("be"), makeString("but"), + makeString("by"), makeString("for"), makeString("if"), makeString("in"), + makeString("into"), makeString("is"), makeString("it"), makeString("no"), + makeString("not"), makeString("of"), makeString("on"), makeString("or"), + makeString("such"), makeString("that"), makeString("the"), makeString("their"), + makeString("then"), makeString("there"), makeString("these"), makeString("they"), + makeString("this"), makeString("to"), makeString("was"), makeString("will"), + makeString("with")))); + + private final CharacterRunAutomaton filter; + private boolean enablePositionIncrements = false; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + + /** + * Create a new MockTokenFilter. + * + * @param input TokenStream to filter + * @param filter DFA representing the terms that should be removed. + * @param enablePositionIncrements true if the removal should accumulate position increments. + */ + public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) { + super(input); + this.filter = filter; + this.enablePositionIncrements = enablePositionIncrements; + } + + @Override + public boolean incrementToken() throws IOException { + // return the first non-stop word found + int skippedPositions = 0; + while (input.incrementToken()) { + if (!filter.run(termAtt.buffer(), 0, termAtt.length())) { + if (enablePositionIncrements) { + posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); + } + return true; + } + skippedPositions += posIncrAtt.getPositionIncrement(); + } + // reached EOS -- return false + return false; + } + + /** + * @see #setEnablePositionIncrements(boolean) + */ + public boolean getEnablePositionIncrements() { + return enablePositionIncrements; + } + + /** + * If true, this Filter will preserve + * positions of the incoming tokens (ie, accumulate and + * set position increments of the removed stop tokens). + */ + public void setEnablePositionIncrements(boolean enable) { + this.enablePositionIncrements = enable; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedComparators.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedComparators.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedComparators.java (revision ) @@ -0,0 +1,276 @@ +package org.apache.lucene.search.exposed; + +import com.ibm.icu.text.RawCollationKey; +import com.ibm.icu.text.RuleBasedCollator; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.util.Comparator; +import java.util.Locale; + +/** + * Wraps different types of comparators. + */ +public class ExposedComparators { + + /** + * Simple atomic specialization of Comparator. Normally used to + * compare terms by treating the arguments as ordinals and performing lookup + * in an underlying reader. + */ + public interface OrdinalComparator { + int compare(int value1, int value2); + } + + public static Comparator localeToBytesRef(final Locale locale) { + return locale == null ? new NaturalComparator() + : collatorToBytesRef(getCollator(locale)); + } + + /** + * Creates a Collator that ignores punctuation and whitespace, mimicking the + * Sun/Oracle default Collator. + * @param locale defines the compare-rules. + * @return a Collator ready for use. + */ + public static Collator getCollator(Locale locale) { + Collator collator = Collator.getInstance(locale); + if (collator instanceof RuleBasedCollator) { + // Treat spaces as normal characters + ((RuleBasedCollator)collator).setAlternateHandlingShifted(false); + } + return collator; + } + + public static Comparator collatorToBytesRef( + final Collator collator) { + return collator == null ? new NaturalComparator() + : new BytesRefWrappedCollator(collator); + } + public static final class BytesRefWrappedCollator implements + Comparator { + private final Collator collator; + public BytesRefWrappedCollator(Collator collator) { + this.collator = collator; + } + public int compare(BytesRef o1, BytesRef o2) { + return collator.compare(o1.utf8ToString(), o2.utf8ToString()); + } + public Collator getCollator() { + return collator; + } + } + public static final class NaturalComparator implements Comparator { + public int compare(BytesRef o1, BytesRef o2) { + return o1.compareTo(o2); // TODO: Consider null-case + } + } + + /** + * @param provider the provider of BytesRefs. + * @param comparator compares two BytesRef against each other. + * @return a comparator that compares ordinals resolvable by provider by + * looking up the BytesRefs values and feeding them to the comparator. + */ + public static OrdinalComparator wrap(TermProvider provider, + Comparator comparator) { + return new BytesRefWrapper(provider, comparator); + } + + /** + * @param provider the provider of BytesRefs. + * @param map indirect map from logical index to term. + * @param comparator compares two BytesRef against each other. + * @return a comparator that compares ordinals resolvable by provider by + * looking up the BytesRefs values and feeding them to the comparator. + * The BytesRefs are requested by calling getTerm with map[value]. + */ + public static OrdinalComparator wrapIndirect( + TermProvider provider, int[] map, Comparator comparator) { + return new IndirectBytesRefWrapper(provider, map, comparator); + } + + /** + * @param provider the provider of BytesRefs. + * @param map indirect map from logical index to ordinal. + * @return a comparator that compares ordinals resolvable by provider by + * looking up the CollatorKeys and comparing them. + * The keys are requested by calling get with map[value]. + */ + public static OrdinalComparator wrapIndirect( + CachedCollatorKeyProvider provider, int[] map) { + return new IndirectCollatorKeyWrapper(provider, map); + } + + /** + * Creates a comparator that relies on a list of ExposedTuples to get the + * BytesRef for comparison. + * @param backingTuples the tuples with the terms to use for comparisons. + * @param comparator compares two BytesRef against each other. + * @return a comparator that compares indexes relative to the backingTuples. + * The BytesRefs are requested by calling {@code backingTuples[index].term}. + */ + public static OrdinalComparator wrapBacking( + ExposedTuple[] backingTuples, Comparator comparator) { + return new BackingTupleWrapper(backingTuples, comparator); + } + + public static OrdinalComparator wrapBacking(RawCollationKey[] backingKeys) { + return new BackingCollatorWrapper(backingKeys); + } + + /** + * Compares the given ordinals directly. + */ + public static final class DirectComparator implements OrdinalComparator { + public final int compare(final int value1, final int value2) { + return value1-value2; + } + } + + /** + * Uses the given values as indexes in a backing int[], comparing the values + * from the int[] directly. If the values are the same, natural integer order + * for the values are used. + */ + public static final class IndirectComparator implements OrdinalComparator { + private final int[] values; + + public IndirectComparator(int[] values) { + this.values = values; + } + + public final int compare(final int value1, final int value2) { + final int diff = values[value1]-values[value2]; + return diff == 0 ? value2 - value1 : diff; + } + } + + /** + * Like {@link IndirectComparator} but in reverse order. + */ + public static final class ReverseIndirectComparator + implements OrdinalComparator { + private final int[] values; + + public ReverseIndirectComparator(int[] values) { + this.values = values; + } + + public final int compare(final int value1, final int value2) { + final int diff = values[value2]-values[value1]; + return diff == 0 ? value2 - value1 : diff; + } + } + + private static class BytesRefWrapper implements OrdinalComparator { + private final TermProvider provider; + private final Comparator comparator; + + public BytesRefWrapper( + TermProvider provider, Comparator comparator) { + this.comparator = comparator; + this.provider = provider; + } + + public final int compare(final int value1, final int value2) { + try { + return comparator == null ? + provider.getTerm(value1).compareTo(provider.getTerm(value2)) : + comparator.compare( + provider.getTerm(value1), provider.getTerm(value2)); + } catch (IOException e) { + throw new RuntimeException( + "IOException encountered while comparing terms for the ordinals " + + value1 + " and " + value2, e); + } + } + } + + private static class IndirectBytesRefWrapper implements OrdinalComparator { + private final TermProvider provider; + private final int[] map; + private final Comparator comparator; + + public IndirectBytesRefWrapper( + TermProvider provider, int[] map, Comparator comparator) { + this.provider = provider; + this.map = map; + this.comparator = comparator; + } + + public final int compare(final int value1, final int value2) { + try { + return comparator.compare( + provider.getTerm(map[value1]), provider.getTerm(map[value2])); + } catch (IOException e) { + throw new RuntimeException( + "IOException encountered while comparing terms for the ordinals " + + value1 + " and " + value2, e); + } + } + } + + private static class IndirectCollatorKeyWrapper implements OrdinalComparator { + private final CachedCollatorKeyProvider provider; + private final int[] map; + + public IndirectCollatorKeyWrapper( + CachedCollatorKeyProvider provider, int[] map) { + this.provider = provider; + this.map = map; + } + + // TODO: Consider null as value + public final int compare(final int value1, final int value2) { + try { + return provider.get(map[value1]).compareTo(provider.get(map[value2])); + } catch (IOException e) { + throw new RuntimeException( + "IOException encountered while comparing keys for the ordinals " + + value1 + " and " + value2, e); + } + } + } + + private static class BackingTupleWrapper implements OrdinalComparator { + public ExposedTuple[] backingTerms; // Updated from the outside + private final Comparator comparator; + + private BackingTupleWrapper( + ExposedTuple[] backingTerms, Comparator comparator) { + this.backingTerms = backingTerms; + this.comparator = comparator; + } + + public int compare(int v1, int v2) { + try { + return comparator == null ? + backingTerms[v1].term.compareTo(backingTerms[v2].term) : + comparator.compare(backingTerms[v1].term, backingTerms[v2].term); + } catch (NullPointerException e) { + // TODO: We should not fail silently here, but how do we notify? + return 0; + } + } + } + + private static class BackingCollatorWrapper implements OrdinalComparator { + public RawCollationKey[] backingKeys; // Updated from the outside + + public BackingCollatorWrapper(RawCollationKey[] backingKeys) { + this.backingKeys = backingKeys; + } + + public int compare(int v1, int v2) { + try { + return backingKeys[v1].compareTo(backingKeys[v2]); + } catch (NullPointerException e) { + return 0; + } + } + + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.java (revision ) @@ -0,0 +1,410 @@ +package org.apache.lucene.search.exposed.facet.request; + +import org.apache.lucene.search.exposed.facet.ParseHelper; + +import javax.xml.stream.*; +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * A specifications of groups of fields with sort order, max tags etc. + * Used for specifying how to generate the internal facet structure as well as + * the concrete user-request. The vocabulary is taken primarily from + * http://wiki.apache.org/solr/SimpleFacetParameters + *

+ * See FacetRequest.xsd and FacetRequest.xml for syntax. + *

+ * The facet request works with overriding params. FacetRequest is top level, + * then follows FacetRequestGroups. + */ +public class FacetRequest { + public static final String NAMESPACE = + "http://lucene.apache.org/exposed/facet/request/1.0"; + + public enum GROUP_ORDER {count, index, locale; + public static GROUP_ORDER fromString(String order) { + if (count.toString().equals(order)) { + return count; + } + if (index.toString().equals(order)) { + return index; + } + if (locale.toString().equals(order)) { + return locale; + } + throw new IllegalArgumentException("The order was '" + order + + "' where only " + GROUP_ORDER.count+ ", " + GROUP_ORDER.index + + " and " + GROUP_ORDER.locale + " is allowed"); + } + } + + public static final GROUP_ORDER DEFAULT_GROUP_ORDER = GROUP_ORDER.count; + public static final boolean DEFAULT_REVERSE = false; + public static final int DEFAULT_MAXTAGS = 20; + public static final int DEFAULT_MINCOUNT = 0; + public static final int DEFAULT_OFFSET = 0; + public static final String DEFAULT_PREFIX = ""; + private static final boolean DEFAULT_HIERARCHICAL = false; + + private static final XMLInputFactory xmlFactory; + static { + xmlFactory = XMLInputFactory.newInstance(); + xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, true); + } + private static final XMLOutputFactory xmlOutFactory; + static { + xmlOutFactory = XMLOutputFactory.newInstance(); + } + + // groups: group* + // group: fieldName* sort offset limit mincount prefix + // sort: count, index, custom (comparator) + private String stringQuery; + private final List groups; + + // Defaults + private GROUP_ORDER order = DEFAULT_GROUP_ORDER; + private boolean reverse = DEFAULT_REVERSE; + private String locale = null; + private int maxTags = DEFAULT_MAXTAGS; + private int minCount = DEFAULT_MINCOUNT; + private int offset = DEFAULT_OFFSET; + private String prefix = DEFAULT_PREFIX; + private boolean hierarchical = DEFAULT_HIERARCHICAL; + private String delimiter = FacetRequestGroup.DEFAULT_DELIMITER; + private int levels = FacetRequestGroup.DEFAULT_LEVELS; + private String startPath = null; + + public FacetRequest(String stringQuery, List groups) { + this.stringQuery = stringQuery; + this.groups = groups; + } + + public FacetRequest(String stringQuery) { + this.stringQuery = stringQuery; + this.groups = new ArrayList(); + } + + /** + * @param request A facet request in XML as specified in FacetRequest.xsd. + * @return a facet request in Object-form. + * @throws XMLStreamException if the request could not be parsed. + */ + public static FacetRequest parseXML(String request) + throws XMLStreamException { + XMLStreamReader reader = + xmlFactory.createXMLStreamReader(new StringReader(request)); + List groups = null; + + String stringQuery = null; + GROUP_ORDER order = DEFAULT_GROUP_ORDER; + boolean reverse = DEFAULT_REVERSE; + String locale = null; + int maxTags = DEFAULT_MAXTAGS; + int minCount = DEFAULT_MINCOUNT; + int offset = DEFAULT_OFFSET; + String prefix = DEFAULT_PREFIX; + boolean hierarchical = DEFAULT_HIERARCHICAL; + int levels = FacetRequestGroup.DEFAULT_LEVELS; + String delimiter = FacetRequestGroup.DEFAULT_DELIMITER; + String startPath = null; + + reader.nextTag(); + if (!ParseHelper.atStart(reader, "facetrequest")) { + throw new XMLStreamException( + "Could not locate start tag in " + request); + } + for (int i = 0 ; i < reader.getAttributeCount() ; i++) { + String attribute = reader.getAttributeLocalName(i); + String value = reader.getAttributeValue(i); + if ("order".equals(attribute)) { + order = GROUP_ORDER.fromString(value); + } else if ("reverse".equals(attribute)) { + reverse = Boolean.parseBoolean(value); + } else if ("locale".equals(attribute)) { + locale = value; + } else if ("maxtags".equals(attribute)) { + maxTags = ParseHelper.getInteger(request, "maxtags", value); + } else if ("mincount".equals(attribute)) { + minCount = ParseHelper.getInteger(request, "mincount", value); + } else if ("offset".equals(attribute)) { + offset = ParseHelper.getInteger(request, "offset", value); + } else if ("prefix".equals(attribute)) { + prefix = value; + } else if ("hierarchical".equals(attribute)) { + hierarchical = Boolean.parseBoolean(value); + } else if ("levels".equals(attribute)) { + levels = ParseHelper.getInteger(request, "levels", value); + } else if ("delimiter".equals(attribute)) { + delimiter = value; + } else if ("startPath".equals(attribute)) { + startPath = value; + } + } + + while ((stringQuery == null || groups == null) + && reader.nextTag() != XMLStreamReader.END_DOCUMENT) { + if (ParseHelper.atStart(reader, "query")) { + reader.next(); + stringQuery = reader.getText(); + reader.nextTag(); // END_ELEMENT + } else if (ParseHelper.atStart(reader, "groups")) { + groups = resolveGroups(reader, request, order, reverse, locale, + maxTags, minCount, offset, prefix, hierarchical, + levels, delimiter, startPath); + } else { + reader.nextTag(); // Ignore and skip to END_ELEMENT + } + } + FacetRequest fr = new FacetRequest(stringQuery, groups); + fr.setOrder(order); + fr.setReverse(reverse); + fr.setLocale(locale); + fr.setMaxTags(maxTags); + fr.setMinCount(minCount); + fr.setOffset(offset); + fr.setPrefix(prefix); + fr.setHierarchical(hierarchical); + fr.setLevels(levels); + fr.setDelimiter(delimiter); + fr.setStartPath(startPath); + return fr; + } + + private static List resolveGroups( + XMLStreamReader reader, String request, GROUP_ORDER order, + boolean reverse, String locale, int maxTags, int minCount, int offset, + String prefix, boolean hierarchical, int levels, String delimiter, + String startPath) throws XMLStreamException { + List groups = new ArrayList(); + while (reader.nextTag() != XMLStreamReader.END_DOCUMENT + && !ParseHelper.atEnd(reader, "groups")) { // Not END_ELEMENT for groups + if (ParseHelper.atStart(reader, "group")) { + groups.add(new FacetRequestGroup( + reader, request, order, reverse, locale, maxTags, minCount, offset, + prefix, hierarchical, levels, delimiter, startPath)); + } else { + reader.nextTag(); // Ignore and skip to END_ELEMENT + } + } + if (groups.size() == 0) { + throw new XMLStreamException("No groups defined for " + request); + } + return groups; + } + + // TODO: Consider sorting the groups and fields to ensure compatibility + // with request where the order is changed + /** + * Produces a key created from the build-specific properties of the request. + * This includes stringQuery, group name, fields and comparator. It does not + * include extraction-specific things such as sort, mincount, offset and + * prefix. The key can be used for caching of the counts from a request. + * @return a key generated from the build-properties. + */ + public String getBuildKey() { + StringWriter sw = new StringWriter(); + sw.append("facetrequest(").append("query(").append(stringQuery); + sw.append("), "); + writeGroupKey(sw); + sw.append(")"); + return sw.toString(); + } + + public String toXML() { + StringWriter sw = new StringWriter(); + try { + XMLStreamWriter out = xmlOutFactory.createXMLStreamWriter(sw); + out.writeStartDocument("utf-8", "1.0"); + out.writeCharacters("\n"); + toXML(out); + out.writeEndDocument(); + out.flush(); + } catch (XMLStreamException e) { // Getting here means error in the code + throw new RuntimeException("Unable to create XML", e); + } + return sw.toString(); + } + public void toXML(XMLStreamWriter out) throws XMLStreamException { + out.setDefaultNamespace(NAMESPACE); + out.writeStartElement("facetrequest"); + out.writeDefaultNamespace(NAMESPACE); + out.writeAttribute("order", order.toString()); + if (locale != null) { + out.writeAttribute("locale", locale); + } + out.writeAttribute("maxtags", Integer.toString(maxTags)); + out.writeAttribute("mincount", Integer.toString(minCount)); + out.writeAttribute("offset", Integer.toString(offset)); + out.writeAttribute("prefix", prefix); + out.writeAttribute("hierarchical", Boolean.toString(hierarchical)); + out.writeAttribute("levels", Integer.toString(levels)); + out.writeAttribute("delimiter", delimiter); + if (startPath != null) { + out.writeAttribute("startPath", startPath); + } + out.writeCharacters("\n "); + + out.writeStartElement("query"); + out.writeCharacters(stringQuery); + out.writeEndElement(); + out.writeCharacters("\n "); + + out.writeStartElement("groups"); + out.writeCharacters("\n"); + for (FacetRequestGroup group: groups) { + group.toXML(out); + } + out.writeCharacters(" "); + out.writeEndElement(); // groups + out.writeCharacters("\n"); + + out.writeEndElement(); // facetrequest + out.writeCharacters("\n"); + out.flush(); + } + + /** + * Produces a key generated from the group-specific properties of the request. + * This includes group name, fields and comparator. + * The key is normally used for caching TermProviders. + * @return a key generated from the group-specific properties of the request. + */ + public String getGroupKey() { + StringWriter sw = new StringWriter(); + writeGroupKey(sw); + return sw.toString(); + } + + public List getGroups() { + return groups; + } + + /** + * Creates a group based on the default values and adds it to this request. + * @param groupName the name for the group. + * @return the newly created group. + */ + public FacetRequestGroup createGroup(String groupName) { + FacetRequestGroup group = new FacetRequestGroup( + groupName, order, reverse, locale, maxTags, minCount, offset, prefix, + hierarchical, levels, delimiter, startPath); + groups.add(group); + return group; + } + + private void writeGroupKey(StringWriter sw) { + sw.append("groups("); + boolean first = true; + for (FacetRequestGroup group: groups) { + if (first) { + first = false; + } else { + sw.append(", "); + } + sw.append(group.getBuildKey()); + } + sw.append(")"); + } + + /* Mutators */ + + public GROUP_ORDER getOrder() { + return order; + } + + public void setOrder(GROUP_ORDER order) { + this.order = order; + } + + public String getLocale() { + return locale; + } + + public void setReverse(boolean reverse) { + this.reverse = reverse; + } + + public void setLocale(String locale) { + this.locale = locale; + } + + public int getMaxTags() { + return maxTags; + } + + public void setMaxTags(int maxTags) { + this.maxTags = maxTags; + } + + public int getMinCount() { + return minCount; + } + + public void setMinCount(int minCount) { + this.minCount = minCount; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public boolean isHierarchical() { + return hierarchical; + } + + public void setHierarchical(boolean hierarchical) { + this.hierarchical = hierarchical; + } + + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public int getLevels() { + return levels; + } + + public void setLevels(int levels) { + this.levels = levels; + } + + public String getStartPath() { + return startPath; + } + + public void setStartPath(String startPath) { + this.startPath = startPath; + } + + public String getQuery() { + return stringQuery; + } + + /** + * Warning: Changing the query is not recommended as it is used af key in + * cahce pools. + * @param query the new query. + */ + public void setQuery(String query) { + this.stringQuery = query; + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java (revision ) @@ -0,0 +1,225 @@ +/* $Id:$ + * + * The Summa project. + * Copyright (C) 2005-2010 The State and University Library + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldComparatorSource; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.util.*; + +/** + * Custom sorter that uses the Exposed framework. Trade-offs are slow first-time + * use, fast subsequent use and low memory footprint. + */ +// TODO: Add adjustable sortNullFirst +public class ExposedFieldComparatorSource extends FieldComparatorSource { + private final IndexReader reader; + private final Comparator comparator; + private final String comparatorID; + private final boolean sortNullFirst; + + // TODO: Figure out how to avoid the need for re-creation upon re-open + + /** + * Creates a field comparator source with the given comparator. It is + * recommended to use this constructor instead of constructing with a locale + * an to provide a fast comparator, such as the ICU Collator. + *

+ * The class {@link ExposedComparators} provides easy wrapping of Collators + * and other comparators. + * @param reader the reader to sort for. + * @param comparator a custom comparator. + * @param comparatorID an id for the comparator. + * @param sortNullFirst if true, documents without content in the sort field + * are sorted first. + */ + public ExposedFieldComparatorSource( + IndexReader reader, Comparator comparator, String comparatorID, + boolean sortNullFirst) { + this.reader = reader; + this.comparatorID = comparatorID; + this.comparator = comparator; + this.sortNullFirst = sortNullFirst; + } + + /** + * Creates a field comparator source with a Collator generated from the given + * locale using {@link ExposedComparators}. By default this means Java's + * build-in Collator, which is rather slow. + * @param reader the reader to sort for. + * @param locale the locale to use for constructing the Collator. + */ + public ExposedFieldComparatorSource(IndexReader reader, Locale locale) { + this(reader, + ExposedComparators.localeToBytesRef(locale), + locale == null ? ExposedRequest.LUCENE_ORDER : locale.toString(), + false); + } + + /** + * @param fieldname the field to sort on. If a group of fields is wanted, + * field names must be separated by {@code ;}. + * Example: {@code foo;bar;zoo}. Note that grouping treats the terms + * from the separate fields at the same level and that there should + * only be a single term i total for the given fields for a single + * document. Grouping is not normally used at this level. + * @param numHits the number of hits to return. + * @param sortPos ? + * @param reversed reversed sort order. + * @return a comparator based on the given parameters. + * @throws IOException if the + */ + @Override + public FieldComparator newComparator( + String fieldname, int numHits, int sortPos, boolean reversed) + throws IOException { + List fieldNames = Arrays.asList(fieldname.split(":")); + return new ExposedFieldComparator( + reader, fieldname, fieldNames, numHits, sortPos, reversed); + } + + private class ExposedFieldComparator extends FieldComparator { + private final TermProvider provider; + + private final int undefinedTerm = -1; + private final PackedInts.Reader docOrder; + + private final String groupName; + private final List fieldNames; + private final int numHits; + private final int sortPos; + private final boolean reversed; + private final int factor; // Reverse-indicator + private int docBase = 0; // Added to all incoming docIDs + + private int[] order; // docOrder + private int bottom; // docOrder + + public ExposedFieldComparator( + IndexReader reader, String groupName, List fieldNames, + int numHits, int sortPos, boolean reversed) throws IOException { + this.groupName = groupName; + this.fieldNames = fieldNames; + this.provider = ExposedCache.getInstance().getProvider( + reader, groupName, fieldNames, comparator, comparatorID); + + this.numHits = numHits; + this.sortPos = sortPos; + // TODO: Whys is it wrong to reverse here when it is a parameter? + this.reversed = false; //reversed; // Reverse is handled at this level + factor = 1; //reversed ? -1 : 1; + docOrder = provider.getDocToSingleIndirect(); + + order = new int[numHits]; + } + + @Override + public int compare(int slot1, int slot2) { + if (!sortNullFirst) { + int slot1order = order[slot1]; + int slot2order = order[slot2]; + if (slot1order == undefinedTerm) { + return slot2order == undefinedTerm ? 0 : -1; + } else if (slot2order == undefinedTerm) { + return 1; // TODO: Verify factor (reverse) behaviour + } + return factor * (slot1order - slot2order); + //return slot1order - slot2order; + } + // No check for null as null-values are always assigned -1 + return factor * (order[slot1] - order[slot2]); + // return order[slot1] - order[slot2]; + } + + @Override + public void setBottom(int slot) { + bottom = order[slot]; + } + + @Override + public int compareBottom(final int doc) throws IOException { + if (!sortNullFirst) { + try { + final long bottomOrder = bottom; + final long docOrderR = docOrder.get(doc+docBase); + if (bottomOrder == undefinedTerm) { + return docOrderR == undefinedTerm ? 0 : -1; + } else if (docOrderR == undefinedTerm) { + return 1; + } + return (int)(factor * (bottomOrder - docOrderR)); + //return (int)(bottomOrder - docOrderR); + } catch (ArrayIndexOutOfBoundsException e) { + throw new ArrayIndexOutOfBoundsException( + "Exception requesting at index " + (doc+docBase) + + " with docOrder.size==" + docOrder.size() + ", doc==" + doc + + ", docBase==" + docBase + ", reader maxDoc==" + maxDoc); + } + } + // No check for null as null-values are always assigned -1 + return (int)(factor * (bottom - docOrder.get(doc+docBase))); +// return (int)(bottom - docOrder.get(doc+docBase)); + } + + @Override + public void copy(int slot, int doc) throws IOException { + // TODO: Remove this +// System.out.println("Copy called: order[" + slot + "] = " +// + doc + "+" + docBase + " = " + (doc + docBase)); +// System.out.println("docID " + (doc+docBase) + " has term " + provider.getOrderedTerm(doc+docBase).utf8ToString()); + order[slot] = (int)docOrder.get(doc+docBase); + } + long maxDoc; + @Override + public FieldComparator setNextReader( + IndexReader.AtomicReaderContext context) throws IOException { + this.docBase = context.docBase; + maxDoc = context.reader.maxDoc(); + return this; + } + + + private final BytesRef EMPTY = new BytesRef(""); + @Override + public Comparable value(int slot) { + try { // A bit cryptic as we need to handle the case of no sort term + final long resolvedDocOrder = order[slot]; + // TODO: Remove this +/* System.out.println("Resolving docID " + slot + " with docOrder entry " + + resolvedDocOrder + " to term " + + (resolvedDocOrder == undefinedTerm ? "null" :reader.getTermText( + (int)termOrder.get((int)resolvedDocOrder)))); + */ + return resolvedDocOrder == undefinedTerm ? EMPTY : + provider.getOrderedTerm(resolvedDocOrder); + } catch (IOException e) { + throw new RuntimeException( + "IOException while extracting term String", e); + } + } + } + + +} Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestFacetRequest.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestFacetRequest.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestFacetRequest.java (revision ) @@ -0,0 +1,65 @@ +package org.apache.lucene.search.exposed.facet; + +import junit.framework.Test; +import junit.framework.TestSuite; +import junit.framework.TestCase; + +public class TestFacetRequest extends TestCase { + public TestFacetRequest(String name) { + super(name); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + public static Test suite() { + return new TestSuite(TestFacetRequest.class); + } + + public static final String SAMPLE = + "\n" + + "\n" + + "\n" + + " freetext:\"foo <> bar & zoo\"\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + public void testParseXML() throws Exception { +// System.out.println(FacetRequest.parseXML(SAMPLE).toXML()); + // TODO: Check identity + } + + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java (revision ) @@ -0,0 +1,519 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.IdentityReader; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import com.ibm.icu.text.RawCollationKey; +import com.ibm.icu.text.Collator; +import java.util.Iterator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Keeps track of a single field in a single segment. Optimized towards + * SegmentReaders although it can be used with other IndexReaders as long as + * they are leafs (has no sub readers). + *

+ * The provider is lazy meaning that the ordering is only calculated when + * needed. + */ +public class FieldTermProvider extends TermProviderImpl { + private ExposedRequest.Field request; + private PackedInts.Reader order = null; + + + // TODO: Consider letting the cache be a factor of term count + /** + * The minimum number of Terms or ExtendedTerms to hold in cache during + * processing. + * The allocated amount of memory depends on average term size and whether + * CollatorKey-based sorting is used. With an average term-length of 20 + * characters, each cache-entry takes approximately 100 bytes without + * CollatorKey and 300 with CollatorKey. + * + * . + * 10.000 is thus roughly equivalent to either 1 or 3 MB of RAM depending on + * Collator presence. + */ + static int minSortCacheSize = 20000; // 2-6 MB + static double sortCacheFraction = 0.05; // 1M docs = 5-15 MB + static int maxSortCacheSize = 300000; // 30-90 MB + private int sortCacheSize; + + static int mergeChunkFragmentationFactor = 3; // TODO: Experiment with this + static int iteratorCacheSize = 5000; // TODO: Make this relative + static int iteratorReadAhead = 100; + + /** + * If this is true and the given comparator is a + * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized + * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the + * chunk-sort speed. + */ + public static boolean optimizeCollator = true; + + private Terms terms; + private TermsEnum termsEnum; + + public FieldTermProvider(IndexReader reader, int docIDBase, + ExposedRequest.Field request, + boolean cacheTables) throws IOException { + super(checkReader(reader), docIDBase, request.getComparator(), + request.getComparatorID(), + "Field " + request.getField(), cacheTables); + if (reader.getSequentialSubReaders() != null) { + throw new IllegalArgumentException( + "The IndexReader should be a leaf (no sub readers). It contained " + + reader.getSequentialSubReaders().length + " sub readers"); + } + this.request = request; + + terms = reader.fields().terms(request.getField()); + // TODO: Make an OrdTermsEnum to handle Variable Gap and other non-ord-codec +// termsEnum = terms == null ? null : terms.iterator(); // It's okay to be empty + termsEnum = OrdinalTermsEnum.createEnum(reader, request.getField(), 128); + sortCacheSize = getSortCacheSize(reader); + } + + private static IndexReader checkReader(IndexReader reader) { + if (reader.getSequentialSubReaders() != null) { + throw new IllegalArgumentException( + "The reader for a FieldTermProvider must not contain sub readers"); + } + return reader; + } + + public ExposedRequest.Field getRequest() { + return request; + } + + @Override + public String getField(long ordinal) { + return request.getField(); + } + + private static final int ITERATE_LIMIT = 10; + private long lastOrdinalRequest = -1; + @Override + public synchronized BytesRef getTerm(final long ordinal) throws IOException { + if (termsEnum == null) { + throw new IOException("No terms for field " + request.getField() + + " in segment " + getReader() + + ". Requested ordinal was " + ordinal); + } + // TODO: Upstream this simple sequential access optimization + if (lastOrdinalRequest != -1 && lastOrdinalRequest <= ordinal && + ordinal <= lastOrdinalRequest + ITERATE_LIMIT) { + BytesRef term = termsEnum.term(); + while (lastOrdinalRequest != ordinal) { + term = termsEnum.next(); + if (term == null) { + throw new IOException("Unable to locate term for ordinal " + ordinal); + } + lastOrdinalRequest++; + } + lastOrdinalRequest = ordinal; + return new BytesRef(term); + } + + termsEnum.seekExact(ordinal); +/* if (TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) { + throw new IOException("Unable to locate term for ordinal " + ordinal); + }*/ + BytesRef result = termsEnum.term(); + lastOrdinalRequest = ordinal; + return new BytesRef(result); + } + + @Override + public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException { + if (termsEnum == null) { + throw new IOException("No terms or docIDs for field " + request.getField() + + " in segment " + getReader() + + ". Requested ordinal was " + ordinal); + } + if (ordinal != lastOrdinalRequest) { + termsEnum.seekExact(ordinal); + //&& TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) { +// throw new IOException("Unable to locate term for ordinal " + ordinal); + } + + lastOrdinalRequest = ordinal; + // + return termsEnum.docs(getReader().getLiveDocs(), reuse); + } + + @Override + public String getOrderedField(long indirect) throws IOException { + return request.getField(); + } + + @Override + public BytesRef getOrderedTerm(final long indirect) throws IOException { + return indirect == -1 ? null : + getTerm(getOrderedOrdinals().get((int)indirect)); + } + + @Override + public long getUniqueTermCount() throws IOException { + return terms == null ? 0 : + termsEnum instanceof OrdinalTermsEnum ? + ((OrdinalTermsEnum) termsEnum).getTermCount() : + terms.getUniqueTermCount(); + } + + @Override + public long getOrdinalTermCount() throws IOException { + return getUniqueTermCount(); // FieldTermProviders only contains uniques + } + + @Override + public long getMaxDoc() { + return getReader().maxDoc(); + } + + @Override + public int getReaderHash() { + return getReader().hashCode(); + } + + @Override + public int getRecursiveHash() { + return getReader().hashCode(); + } + + @Override + public PackedInts.Reader getOrderedOrdinals() throws IOException { + if (termsEnum == null) { + return ExposedSettings.getMutable(0, 1); + } + + if (order != null) { + return order; + } + PackedInts.Reader newOrder; + if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID()) + || ExposedRequest.FREE_ORDER.equals(request.getComparatorID())) { + newOrder = new IdentityReader((int)getUniqueTermCount()); + } else { + newOrder = sortOrdinals(); + } + + if (cacheTables) { + order = newOrder; + } + return newOrder; + } + + @Override + public Iterator getIterator( + boolean collectDocIDs) throws IOException { + PackedInts.Reader order = getOrderedOrdinals(); + if (collectDocIDs) { + return new TermDocIterator(this, true); + } + this.order = order; + CachedTermProvider cache = new CachedTermProvider( + this, iteratorCacheSize, iteratorReadAhead); + return new TermDocIterator(cache, false); + } + + private PackedInts.Reader sortOrdinals() throws IOException { + int termCount = (int)getOrdinalTermCount(); + int[] ordered = new int[termCount]; + for (int i = 0 ; i < termCount ; i++) { + ordered[i] = i; // Initial order = Lucene + } + + long startTime = System.nanoTime(); + sort(ordered); + long sortTime = (System.nanoTime() - startTime); + + + // TODO: Remove this + if (ExposedSettings.debug) { + System.out.println("Chunk total sort for field " + getField(0) + + ": " + ExposedUtil.time( + "terms", ordered.length, sortTime / 1000000)); + } +/* System.out.println(String.format( + "Sorted %d Terms in %s out of which %s (%s%%) was lookups and " + + "%s (%s%%) was collation key creation. " + + "The cache (%d terms) got %d requests with %d (%s%%) misses", + termCount, nsToString(sortTime), + nsToString(lookupTime), + lookupTime * 100 / sortTime, + nsToString(collatorKeyCreation), + collatorKeyCreation * 100 / sortTime, + sortCacheSize, cacheRequests, + cacheMisses, cacheMisses * 100 / cacheRequests)); + */ + PackedInts.Mutable packed = + ExposedSettings.getMutable(termCount, termCount); + for (int i = 0 ; i < termCount ; i++) { + packed.set(i, ordered[i]); // Save space by offsetting min to 0 + } + return packed; + } + + /* + * Starts by dividing the ordered array in logical chunks, then sorts each + * chunk separately and finishes by merging the chunks. + */ + private void sort(final int[] ordinals) throws IOException { + int chunkSize = Math.max(sortCacheSize, ordinals.length / sortCacheSize); + int chunkCount = (int) Math.ceil(ordinals.length * 1.0 / chunkSize); + + // We do not thread the sort as the caching of Strings is more important + // than processing power. + + // We sort in chunks so the cache is 100% effective + if (optimizeCollator && request.getComparator() instanceof + ExposedComparators.BytesRefWrappedCollator) { +// System.out.println("Using CollatorKey optimized chunk sort"); + optimizedChunkSort(ordinals, chunkSize); + } else { + chunkSort(ordinals, chunkSize); + } + + if (chunkSize >= ordinals.length) { + return; // Only one chunk. No need for merging + } + + // We have sorted chunks. Commence the merging! + + chunkMerge(ordinals, chunkSize, chunkCount); + } + + private void chunkMerge( + int[] ordinals, int chunkSize, int chunkCount) throws IOException { + // Merging up to cache-size chunks requires an efficient way to determine + // the chunk with the lowest value. As locality is not that important with + // all comparables in cache, we use a heap. + // The heap contains an index (int) for all active chunks in the term order + // array. When an index is popped, it is incremented and re-inserted unless + // it is a block start index in which case it is just discarded. + + if (ExposedSettings.debug) { + System.out.println("Beginning merge sort of " + ordinals.length + + " ordinals in " + chunkCount + " chunks"); + } + long mergeTime = System.currentTimeMillis(); + CachedProvider cache; + ExposedComparators.OrdinalComparator indirectComparator; + if (optimizeCollator && request.getComparator() instanceof + ExposedComparators.BytesRefWrappedCollator) { + Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request. + getComparator()).getCollator(); + CachedCollatorKeyProvider keyCache = new CachedCollatorKeyProvider( + this, collator, sortCacheSize, chunkSize-1); + keyCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount / + mergeChunkFragmentationFactor)); + keyCache.setOnlyReadAheadIfSpace(true); + keyCache.setStopReadAheadOnExistingValue(true); + indirectComparator = ExposedComparators.wrapIndirect(keyCache, ordinals); + cache = keyCache; + } else { + CachedTermProvider termCache = new CachedTermProvider( + this, sortCacheSize, chunkSize-1); + // Configure the cache so that read ahead is lower as the access pattern + // to the ordinals are not guaranteed linear + termCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount / + mergeChunkFragmentationFactor)); + termCache.setOnlyReadAheadIfSpace(true); + termCache.setStopReadAheadOnExistingValue(true); + for (int i = 0 ; i < ordinals.length ; i += chunkSize) { + termCache.getTerm(i); // Warm cache + } + indirectComparator = ExposedComparators.wrapIndirect( + termCache, ordinals, request.getComparator()); + cache = termCache; + } + + ExposedPriorityQueue pq = new ExposedPriorityQueue( + indirectComparator, chunkCount); + for (int i = 0 ; i < ordinals.length ; i += chunkSize) { + pq.add(i); + } + + int[] sorted = new int[ordinals.length]; + for (int i = 0 ; i < sorted.length ; i++) { + Integer next = pq.pop(); + if (next == -1) { + throw new IllegalStateException( + "Popping the heap should never return -1"); + } + sorted[i] = ordinals[next]; + cache.release(sorted[i]); // Important for cache read ahead efficiency + if (++next % chunkSize != 0 && next != ordinals.length) { + pq.add(next); + } + } + mergeTime = System.currentTimeMillis() - mergeTime; + if (ExposedSettings.debug) { + System.out.println( + "Merged " + ExposedUtil.time("chunks", chunkCount, mergeTime) + + " aka " + ExposedUtil.time("terms", sorted.length, mergeTime) + + ": " + cache.getStats()); + } + System.arraycopy(sorted, 0, ordinals, 0, sorted.length); + if (ExposedSettings.debug) { + System.out.println("Cache stats for chunkMerge for " + getDesignation() + + ": " + cache.getStats()); +/* System.out.println(String.format( + "Heap merged %d sorted chunks of size %d (cache: %d, total terms: %s)" + + " in %s with %d cache misses (%d combined for both sort passes)", + chunkCount, chunkSize, sortCacheSize, ordinals.length, + nsToString(System.nanoTime() - startTimeHeap), + cacheMisses - oldHeapCacheMisses, cacheMisses - oldCacheMisses));*/ + } + } + + private void chunkSort(int[] ordinals, int chunkSize) throws IOException { + CachedTermProvider cache = new CachedTermProvider( + this, sortCacheSize, chunkSize-1); + // Sort the chunks individually + //long startTimeMerge = System.nanoTime(); + ExposedComparators.OrdinalComparator comparator = + ExposedComparators.wrap(cache, request.getComparator()); + for (int i = 0 ; i < ordinals.length ; i += chunkSize) { + long chunkTime = System.currentTimeMillis(); + cache.getTerm(i); // Tim-sort starts at 1 so we init the read-ahead at 0 + ExposedTimSort.sort( + ordinals, i, Math.min(i + chunkSize, ordinals.length), comparator); + chunkTime = System.currentTimeMillis() - chunkTime; + int percent = (Math.min(i + chunkSize, ordinals.length)-1) *100 + / ordinals.length; + if (ExposedSettings.debug) { + System.out.println("Chunk sorted " + percent + "% " + i + "-" + + (Math.min(i + chunkSize, ordinals.length)-1) + ": " + + ExposedUtil.time("ordinals", Math.min(i + chunkSize, + ordinals.length) - i, chunkTime) + ": "+ cache.getStats()); + } + cache.clear(); + } + } + + // Ordinals _must_ be monotonously increasing + private void optimizedChunkSort( + int[] ordinals, int chunkSize) throws IOException { + Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request. + getComparator()).getCollator(); + CachedTermProvider cache = new CachedTermProvider( + this, sortCacheSize, chunkSize-1); + CollatorPair[] keys = new CollatorPair[chunkSize]; + for (int start = 0 ; start < ordinals.length ; start += chunkSize) { + long chunkTime = System.currentTimeMillis(); + + int end = Math.min(start + chunkSize, ordinals.length); // Exclusive + // Fill + for (int index = start ; index < end ; index++) { + RawCollationKey key = new RawCollationKey(); + keys[index - start] = new CollatorPair( + ordinals[index], collator.getRawCollationKey( + cache.getTerm(ordinals[index]).utf8ToString(), key)); + } + // Sort + ExposedTimSort.sort( + ordinals, start, end, new TimComparator(start, keys)); +/* // Store + for (int index = start ; index < end ; index++) { + ordinals[index] = (int) keys[index - start].ordinal; + }*/ + + chunkTime = System.currentTimeMillis() - chunkTime; + int percent = (int) ((long)end * 100 / ordinals.length); + if (ExposedSettings.debug) { + System.out.println("Chunk sorted " + percent + "% " + + start + "-" + (end-1) + ": " + + ExposedUtil.time("ordinals", end - start, chunkTime) + + ": "+ cache.getStats()); + } + cache.clear(); + } + } + private static final class TimComparator + implements ExposedComparators.OrdinalComparator { + private int start; // Inclusive + private CollatorPair[] keys; + + public TimComparator(int start, CollatorPair[] keys) { + this.start = start; + this.keys = keys; + } + + @Override + public int compare(int value1, int value2) { + return keys[value1 - start].compareTo(keys[value2 - start]); + } + } + + + private static final class CollatorPair implements Comparable{ + private long ordinal; + private RawCollationKey key; + + private CollatorPair(long ordinal, RawCollationKey key) { + this.ordinal = ordinal; + this.key = key; + } + @Override + public int compareTo(CollatorPair o) { + return key.compareTo(o.key); + } + @SuppressWarnings({"UnusedDeclaration"}) + public long getOrdinal() { + return ordinal; + } + } + + /** + * Calculate the sort cache size as a fraction of the total number of + * documents in the reader, + * @param reader the reader to cache. + * @return a cache size that fits in scale to the reader. + */ + private int getSortCacheSize(IndexReader reader) { + final int fraction = (int) (reader.maxDoc() * sortCacheFraction); + return Math.min(maxSortCacheSize, Math.max(minSortCacheSize, fraction)); + } + + public String toString() { + if (order == null) { + return "FieldTermProvider(" + request.getField() + ", no order cached, " + + super.toString() + ")"; + } + return "FieldTermProvider(" + request.getField() + + ", order.length=" + order.size() + + " mem=" + packedSize(order) + ", " + super.toString() + ")"; + } + + @Override + public void transitiveReleaseCaches(int level, boolean keepRoot) { + if (!keepRoot || level > 0) { + order = null; + } + super.transitiveReleaseCaches(level, keepRoot); + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xsd =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xsd (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xsd (revision ) @@ -0,0 +1,159 @@ + + + + + XML Schema for Lucene exposed (LUCENE-2369) facet responses. + + + + + + + + + query: The unmodified query used for the Lucene search to + provide faceting for. + hits: The number of documents used for the current result. + countms: The number of milliseconds used for counting all + possible tags for the given query. + countcached: boolean. If true, the counts for the given query + was already in the cache and was re-used. In that case, + countms states the amount of milliseconds used in the + first run. + countcached: If true, the counting part of the facet generation + was skipped as the cache contained the counts. + totalms: The total number of milliseconds from receival of the + request to full response generation, minus XML + generation. This includes counting. + Note that totalms can be lower than countms id the + counting structure was found in the cache. + + + + + + + + + + + + + + + + + name: The name of this facet. + fields: The fields used for this facet. + order: count = the tags are sorted by occurrences. + Highest number comes first. + index = the tags sorted by index order. + locale = the tags are sorted by the locale given in the + attribute "locale". + The default order is count. + locale: If sort is specified to "locale", this locale is used. + maxtags: The maximum number of tags to return for a facet. + -1 is unlimited and _not_ recommended. + The default is 20. + mincount: The minimum number of occurrences in order for a tag + to be part of the result. + The default is 0. + offset: Where to start extracting tags in the sorted list of + tags. Used for pagination. + The default is 0. + Note: This can be negative when used with "prefix" and + order != count. + prefix: The extraction starts at (the first tag that matches the + prefix) + offset. This cannot be used with count order. + The default is "", meaning the beginning of the sorted + tag list. + + hierarchical: If true, the structure is hierarchical. + levels: The maximum number of levels requested. + delimiter: The delimiter used for splitting tags in levels. + + extractionms: The number of milliseconds spend on extracting + the tags for the facet. This includes sliding + window sorting and tagID to Term resolving. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + potentialtags: The number of tags in this facet for a query that + matches all documents. + totaltags: The number of tags with at least mincount occurrences + for the documents matching the given query. + count: The total number of references from documents matching + the given query to tags at the current level. + totalcount: The total number of references from documents matching + the given query to tags at current and sub levels. + + maxtags: The maxtags from the query. + mincount: The mincount from the query. + mintotalcount: The mintotalcount from the query. + suborder: The suborder from the query. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java (revision ) @@ -0,0 +1,259 @@ +package org.apache.lucene.search.exposed; + +import junit.framework.TestCase; +import org.apache.lucene.index.*; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.LuceneTestCase; + +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.util.*; + +// TODO: Change this to LuceneTestCase but ensure Flex +public class TestFieldTermProvider extends TestCase { +// new File(System.getProperty("java.io.tmpdir"), "exposed_index"); +// new File("/home/te/projects/lucene/exposed_index"); +// new File("/mnt/bulk/exposed_index"); + public static final int DOCCOUNT = 100; + private ExposedHelper helper; + + @Override + public void setUp() throws Exception { + super.setUp(); + // TODO: Figure out how to force Flex the clean way +// CodecProvider.setDefaultCodec("Standard"); + helper = new ExposedHelper(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + helper.close(); + } + + public void testCreateIndex() throws IOException { + helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + } + + public void testSegmentcount() throws IOException { + helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + int subCount = reader.getSequentialSubReaders().length; + assertTrue("The number of segments should be >= 2 but was " + subCount, + subCount >= 2); + reader.close(); + } + + public void testIndexGeneration() throws Exception { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + long termCount = 0; + TermsEnum terms = MultiFields.getFields(reader). + terms(ExposedHelper.ID).iterator(); + while (terms.next() != null) { + assertEquals("The ID-term #" + termCount + " should be correct", + ExposedHelper.ID_FORMAT.format(termCount), + terms.term().utf8ToString()); + termCount++; + } + + assertEquals("There should be the right number of terms", + DOCCOUNT, termCount); + reader.close(); + } + + public void testOrdinalAccess() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexReader segment = reader.getSequentialSubReaders()[0]; + + ArrayList plainExtraction = new ArrayList(DOCCOUNT); + TermsEnum terms = + reader.getSequentialSubReaders()[0].fields().terms("a").iterator(); + while (terms.next() != null) { + plainExtraction.add(terms.term().utf8ToString()); + } + + TermProvider segmentProvider = ExposedFactory.createProvider( + segment, null, Arrays.asList("a"), null, false, "null"); + ArrayList exposedOrdinals = new ArrayList(DOCCOUNT); + for (int i = 0 ; i < segmentProvider.getOrdinalTermCount() ; i++) { + exposedOrdinals.add(segmentProvider.getTerm(i).utf8ToString()); + } + + assertEquals("The two lists of terms should be of equal length", + plainExtraction.size(), exposedOrdinals.size()); + for (int i = 0 ; i < plainExtraction.size() ; i++) { + assertEquals("The term at index " + i + " should be correct", + plainExtraction.get(i), exposedOrdinals.get(i)); + } + reader.close(); + + } + + public void testSegmentTermSort() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexReader segment = reader.getSequentialSubReaders()[0]; + testSegmentTermSort(segment); + reader.close(); + } + + private void testSegmentTermSort(IndexReader segment) throws IOException { + Collator sorter = Collator.getInstance(new Locale("da")); + + ArrayList plainExtraction = new ArrayList(DOCCOUNT); + TermsEnum terms = segment.fields().terms("a").iterator(); + while (terms.next() != null) { + plainExtraction.add(terms.term().utf8ToString()); + System.out.println("Unsorted term #" + (plainExtraction.size() - 1) + ": " + + terms.term().utf8ToString()); + } + Collections.sort(plainExtraction, sorter); + + ExposedRequest.Field request = new ExposedRequest.Field( + "a", ExposedComparators.collatorToBytesRef(sorter), + false, "collator_da"); + FieldTermProvider segmentProvider = + new FieldTermProvider(segment, 0, request, true); + + ArrayList exposedExtraction = new ArrayList(DOCCOUNT); + Iterator ei = segmentProvider.getIterator(false); + int count = 0; + while (ei.hasNext()) { + exposedExtraction.add(ei.next().term.utf8ToString()); + System.out.println("Exposed sorted term #" + count++ + ": " + + exposedExtraction.get(exposedExtraction.size()-1)); + } + + assertEquals("The two lists of terms should be of equal length", + plainExtraction.size(), exposedExtraction.size()); + for (int i = 0 ; i < plainExtraction.size() ; i++) { + assertEquals("The term at index " + i + " should be correct", + plainExtraction.get(i), exposedExtraction.get(i)); + } + } + + public void testTermEnum() throws IOException { + ExposedIOFactory.forceFixedCodec = false; + + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexReader segment = reader.getSequentialSubReaders()[0]; + Collator sorter = Collator.getInstance(new Locale("da")); + + ExposedRequest.Field request = new ExposedRequest.Field( + "a", ExposedComparators.collatorToBytesRef(sorter), + false, "collator_da"); + FieldTermProvider segmentProvider = + new FieldTermProvider(segment, 0, request, true); + ArrayList exposed = + new ArrayList(DOCCOUNT); + Iterator ei = segmentProvider.getIterator(true); + while (ei.hasNext()) { + final ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + exposed.add(new ExposedHelper.Pair( + doc + tuple.docIDBase, tuple.term.utf8ToString(), sorter)); + } + } + } + Collections.sort(exposed); + } + + public void testDocIDMapping() throws IOException { + helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexReader segment = reader.getSequentialSubReaders()[0]; + Collator sorter = Collator.getInstance(new Locale("da")); + + ArrayList plain = + new ArrayList(DOCCOUNT); + for (int docID = 0 ; docID < segment.maxDoc() ; docID++) { + plain.add(new ExposedHelper.Pair( + docID, segment.document(docID).get("a"), sorter)); +// System.out.println("Plain access added " + plain.get(plain.size()-1)); + } + Collections.sort(plain); + + ExposedRequest.Field request = new ExposedRequest.Field( + "a", ExposedComparators.collatorToBytesRef(sorter), + false, "collator_da"); + FieldTermProvider segmentProvider = + new FieldTermProvider(segment, 0, request, true); + + ArrayList exposed = + new ArrayList(DOCCOUNT); + Iterator ei = segmentProvider.getIterator(true); + + while (ei.hasNext()) { + final ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + exposed.add(new ExposedHelper.Pair( + doc + tuple.docIDBase, tuple.term.utf8ToString(), sorter)); + } + } + } + Collections.sort(exposed); + + assertEquals("The two docID->term maps should be of equal length", + plain.size(), exposed.size()); + for (int i = 0 ; i < plain.size() ; i++) { + assertEquals("Mapping #" + i + " should be equal but was " + + plain.get(i) + " vs. " + exposed.get(i), + plain.get(i), exposed.get(i)); + System.out.println("Sorted docID, term #" + i + ": " + plain.get(i)); + } + reader.close(); + } + + // Just tests for non-crashing + public void testDocIDMultiMapping() throws IOException { + helper.createIndex(100, Arrays.asList("a"), 20, 2); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexReader segment = reader.getSequentialSubReaders()[0]; + Collator sorter = Collator.getInstance(new Locale("da")); + + ExposedRequest.Field request = new ExposedRequest.Field( + ExposedHelper.MULTI, ExposedComparators.collatorToBytesRef(sorter), + false, "collator_da"); + FieldTermProvider segmentProvider = + new FieldTermProvider(segment, 0, request, true); + + ArrayList exposed = + new ArrayList(DOCCOUNT); + Iterator ei = segmentProvider.getIterator(true); + + while (ei.hasNext()) { + final ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + exposed.add(new ExposedHelper.Pair( + doc + tuple.docIDBase, tuple.term.utf8ToString(), sorter)); + } + } + } + +/* for (int i = 0 ; i < exposed.size() ; i++) { + System.out.println("Sorted docID, term #" + i + ": " + exposed.get(i)); + }*/ + reader.close(); + } + +} Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/rdf.schema =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/rdf.schema (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/rdf.schema (revision ) @@ -0,0 +1,141 @@ + + + + The RDF Vocabulary (RDF) + This is the RDF Schema for the RDF vocabulary defined in the RDF namespace. + + + + + + + PlainLiteral + The class of plain (i.e. untyped) literal values. + + + + + + + type + The subject is an instance of a class. + + + + + + + Property + The class of RDF properties. + + + + + + Statement + + The class of RDF statements. + + + + + subject + The subject of the subject RDF statement. + + + + + + + predicate + The predicate of the subject RDF statement. + + + + + + + object + The object of the subject RDF statement. + + + + + + + Bag + The class of unordered containers. + + + + + + Seq + The class of ordered containers. + + + + + + Alt + The class of containers of alternatives. + + + + + + value + Idiomatic property used for structured values. + + + + + + + + + List + The class of RDF Lists. + + + + + + nil + The empty list, with no items in it. If the rest of a list is nil then the list has no more items in it. + + + + + first + The first item in the subject RDF list. + + + + + + + rest + The rest of the subject RDF list after the first item. + + + + + + + + XMLLiteral + The class of XML literal values. + + + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java (revision ) @@ -0,0 +1,51 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; + +public final class MockVariableLengthPayloadFilter extends TokenFilter { + private static final int MAXLENGTH = 129; + + private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); + private final Random random; + private final byte[] bytes = new byte[MAXLENGTH]; + private final Payload payload; + + public MockVariableLengthPayloadFilter(Random random, TokenStream in) { + super(in); + this.random = random; + this.payload = new Payload(bytes); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + random.nextBytes(bytes); + payload.setData(bytes, 0, random.nextInt(MAXLENGTH)); + payloadAtt.setPayload(payload); + return true; + } else { + return false; + } + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision ) @@ -0,0 +1,94 @@ +package org.apache.lucene.analysis; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.index.Payload; + +import java.io.IOException; +import java.io.Reader; + + +/** + * + * + **/ +public final class MockPayloadAnalyzer extends Analyzer { + + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + TokenStream result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); + return new MockPayloadFilter(result, fieldName); + } +} + + +/** + * + * + **/ +final class MockPayloadFilter extends TokenFilter { + String fieldName; + + int pos; + + int i; + + final PositionIncrementAttribute posIncrAttr; + final PayloadAttribute payloadAttr; + final CharTermAttribute termAttr; + + public MockPayloadFilter(TokenStream input, String fieldName) { + super(input); + this.fieldName = fieldName; + pos = 0; + i = 0; + posIncrAttr = input.addAttribute(PositionIncrementAttribute.class); + payloadAttr = input.addAttribute(PayloadAttribute.class); + termAttr = input.addAttribute(CharTermAttribute.class); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); + int posIncr; + if (i % 2 == 1) { + posIncr = 1; + } else { + posIncr = 0; + } + posIncrAttr.setPositionIncrement(posIncr); + pos += posIncr; + i++; + return true; + } else { + return false; + } + } + + @Override + public void reset() throws IOException { + super.reset(); + i = 0; + pos = 0; + } +} + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/Pool.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/Pool.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/Pool.java (revision ) @@ -0,0 +1,29 @@ +package org.apache.lucene.search.exposed.facet; + +import java.util.ArrayList; +import java.util.List; + +/** + * Simple generic pool that creates new elements if empty. The caller is + * responsible for clearing the acquired elements. + */ +public abstract class Pool { + private List elements = new ArrayList(10); + + public abstract T createElement(); + + public synchronized T acquire() { + if (elements.size() == 0) { + return createElement(); + } + return elements.remove(elements.size()-1); + } + + public synchronized void release(T element) { + elements.add(element); + } + + public void clear() { + elements.clear(); + } +} Index: solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetQueryComponent.java =================================================================== --- solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetQueryComponent.java (revision ) +++ solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetQueryComponent.java (revision ) @@ -0,0 +1,234 @@ +package org.apache.solr.exposed; + +import org.apache.lucene.search.exposed.facet.CollectorPool; +import org.apache.lucene.search.exposed.facet.CollectorPoolFactory; +import org.apache.lucene.search.exposed.facet.FacetResponse; +import org.apache.lucene.search.exposed.facet.TagCollector; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.handler.component.QueryComponent; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.apache.solr.exposed.ExposedFacetParams.*; + +/** + * Wrapper for the Exposed-patch for Lucene, providing fast, low-memory, + * expensive-startup hierarchical faceting. Arguments are mimicked from + * http://wiki.apache.org/solr/SimpleFacetParameters + *

+ * efacet.query: Lucene query parser syntax query used for faceting. + * Cannot be overridden for specific fields. + */ +public class ExposedFacetQueryComponent extends QueryComponent { + public static final Logger log = + LoggerFactory.getLogger(SolrResourceLoader.class); + + public static final int DEFAULT_COLLECTOR_POOLS = 12; + public static final int DEFAULT_COLLECTOR_FILLED = 2; + public static final int DEFAULT_COLLECTOR_FRESH = 2; + + private CollectorPoolFactory poolFactory; + + @Override + public void init(NamedList args) { + int cPools = DEFAULT_COLLECTOR_POOLS; + int cFresh = DEFAULT_COLLECTOR_FRESH; + int cFilled= DEFAULT_COLLECTOR_FILLED; + if (args.get("pools") != null) { + NamedList pools =(NamedList)args.get("pools"); + for (int i = 0 ; i < pools.size() ; i++) { + if ("pools".equals(pools.getName(i))) { + cPools = Integer.parseInt((String)pools.getVal(i)); + } else + if ("fresh".equals(pools.getName(i))) { + cFresh = Integer.parseInt((String)pools.getVal(i)); + } else + if ("filled".equals(pools.getName(i))) { + cFilled = Integer.parseInt((String)pools.getVal(i)); + } + } + } + poolFactory = new CollectorPoolFactory(cPools, cFilled, cFresh); + } + + @Override + public void prepare(ResponseBuilder rb) throws IOException { + if (rb.req.getParams().getBool(EFACET, false)) { + // TODO: Check if the query is already cached + rb.setNeedDocSet( true ); + //rb.doFacets = true; + } + } + + @Override + public void process(ResponseBuilder rb) throws IOException { + if (poolFactory == null) { + throw new IllegalStateException("CollectorPoolFactory not initialized. " + + "Init must be called before process"); + } + + SolrQueryRequest req = rb.req; + SolrQueryResponse rsp = rb.rsp; + SolrParams params = req.getParams(); + if (!"true".equals(params.get(EFACET))) { + return; // Not an exposed facet request + } + + FacetRequest eReq = createRequest(params); + if (eReq == null) { + return; // Not a usable request for exposed facets + } + + CollectorPool collectorPool; + try { + collectorPool = poolFactory.acquire( + req.getSearcher().getIndexReader(), eReq); + } catch (IOException e) { + throw new RuntimeException( + "Unable to acquire a CollectorPool for " + eReq, e); + } + TagCollector tagCollector = collectorPool.acquire(eReq.getBuildKey()); + + if (tagCollector.getQuery() == null) { // Not cached + tagCollector.collect(rb.getResults().docSet.getBits()); + } +/* if (tagCollector.getQuery() == null) { // Not cached + try { + QParser qp = QParser.getParser(eReq.getQuery(), null, req); + System.out.println( + "Query parsed '" + eReq.getQuery() + "' to " + qp.parse()); + req.getSearcher().search(qp.parse(), tagCollector); + } catch (ParseException e) { + throw new IllegalArgumentException( + "Unable to parse '" + eReq.getQuery() + "'", e); + } + }*/ + FacetResponse facetResponse; + try { + facetResponse = tagCollector.extractResult(eReq); + } catch (IOException e) { + throw new RuntimeException( + "Unable to extract response from TagCollector", e); + } + collectorPool.release(eReq.getBuildKey(), tagCollector); + exposedToSolr(facetResponse, rsp); + } + + // See http://wiki.apache.org/solr/SimpleFacetParameters#Examples + private void exposedToSolr(FacetResponse fr, SolrQueryResponse rsp) { + NamedList res = new SimpleOrderedMap(); + NamedList fields = new SimpleOrderedMap(); // Skip groups + + for (FacetResponse.Group group: fr.getGroups()) { + if (group.isHierarchical()) { + exposedHierarchicalGroupToSolr(group, fields); + } else { + NamedList field = new SimpleOrderedMap(); + for (FacetResponse.Tag tag: group.getTags().getTags()) { + field.add(tag.getTerm(), tag.getCount()); + } + fields.add(group.getFieldsStr(), field); + } + } + res.add(EFACET + "_fields", fields); + rsp.add(EFACET + "_counts", res); + } + + private void exposedHierarchicalGroupToSolr( + FacetResponse.Group group, NamedList fields) { + NamedList content = new SimpleOrderedMap(); + content.add("field", group.getFieldsStr()); + FacetResponse.TagCollection tags = group.getTags(); + content.add("paths", expandHierarchical( + tags, group.getRequest().getDelimiter(), 0)); + fields.add(group.getFieldsStr(), content); + } + + private NamedList expandHierarchical( + FacetResponse.TagCollection tags, String delimiter, int level) { + NamedList content = new SimpleOrderedMap(); + content.add("recursivecount", tags.getTotalCount()); + content.add("potentialtags", tags.getPotentialTags()); + content.add("totaltags", tags.getTotalTags()); + content.add("count", tags.getCount()); + content.add("level", level++); + NamedList cTags = new SimpleOrderedMap(); + for (FacetResponse.Tag tag: tags.getTags()) { + NamedList cTag = new SimpleOrderedMap(); + cTag.add("count", tag.getCount()); +// cTag.add("path", tag.getTerm()); // TODO: Calculate full path + if (tag.getSubTags() != null) { + cTag.add("sub", + expandHierarchical(tag.getSubTags(), delimiter, level)); + } + String[] tokens = tag.getTerm().split(delimiter); + cTags.add(tokens[tokens.length-1], cTag); + } + content.add("sub", cTags); + return content; + } + + private FacetRequest createRequest(SolrParams params) { + String query = params.get("q"); + if (query == null) { + throw new IllegalArgumentException( + "The parameter 'q' must be specified"); + } + String[] fieldNames = params.getParams(EFACET_FIELD); + if (fieldNames == null) { + throw new IllegalArgumentException( + "At least one field must be specified with " + EFACET_FIELD); + } + + FacetRequest eReq = new FacetRequest(query); + // Default values + eReq.setMaxTags(params.getInt(EFACET_LIMIT, eReq.getMaxTags())); + eReq.setOffset(params.getInt(EFACET_OFFSET, eReq.getOffset())); + eReq.setMinCount(params.getInt(EFACET_MINCOUNT, eReq.getMinCount())); + String sort = params.get(EFACET_SORT, EFACET_SORT_COUNT); // Count is def. + Boolean reverse = params.getBool(EFACET_REVERSE, false); + if (EFACET_SORT_INDEX.equals(sort)) { + eReq.setOrder(FacetRequest.GROUP_ORDER.index); + } else if (EFACET_SORT_LOCALE.equals(sort)) { + eReq.setOrder(FacetRequest.GROUP_ORDER.locale); + String locale =params.get(EFACET_SORT_LOCALE_VALUE); + if (locale != null) { + eReq.setLocale(locale); + } + } else { // EFACET_SORT_COUNT.equals(sort) + eReq.setOrder(FacetRequest.GROUP_ORDER.count); // default + } + eReq.setHierarchical(params.getBool(EFACET_HIERARCHICAL, false)); + eReq.setDelimiter(params.get( + EFACET_HIERARCHICAL_DELIMITER, EFACET_HIERARCHICAL_DELIMITER_DEFAULT)); + eReq.setLevels(params.getInt( + EFACET_HIERARCHICAL_LEVELS, Integer.MAX_VALUE)); + + eReq.setReverse(reverse); + + // TODO: Add hierarchical + // TODO: Consider adding grouping + + // Specific fields + for (String fieldName: fieldNames) { + // TODO: Handle field-specific settings + eReq.createGroup(fieldName); + } + return eReq; + } + + // key("efacet.offset", "title") -> "efacet.title.offset" + private String key(String majorKey, String fieldName) { + return majorKey.substring(0, EFACET.length()) + "." + fieldName + + majorKey.substring(EFACET.length(), majorKey.length()); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/SubtagsConstraints.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/SubtagsConstraints.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/SubtagsConstraints.java (revision ) @@ -0,0 +1,29 @@ +package org.apache.lucene.search.exposed.facet.request; + +public interface SubtagsConstraints { + int getMaxTags(); + int getMinCount(); + int getMinTotalCount(); + SUBTAGS_ORDER getSubtagsOrder(); + + /** + * + * @return a SubtagsConstraints for the next level or the current definer if there + * are no defined sub level. + */ + SubtagsConstraints getDeeperLevel(); + + enum SUBTAGS_ORDER {count, base; + public static SUBTAGS_ORDER fromString(String order) { + if (count.toString().equals(order)) { + return count; + } + if (base.toString().equals(order)) { + return base; + } + throw new IllegalArgumentException("The order was '" + order + + "' where only " + SUBTAGS_ORDER.count + + " and " + SUBTAGS_ORDER.base + " is allowed"); + } + } +} Index: lucene/contrib/exposed/build.xml =================================================================== --- lucene/contrib/exposed/build.xml (revision ) +++ lucene/contrib/exposed/build.xml (revision ) @@ -0,0 +1,61 @@ + + + + + + + + Ordinal-oriented index-wide search-time sorting and (hierarchical) faceting. High performance and low memory footprint at the cost of longer startup time. + + + + + + + + + + + + + + + + + + + + + + + + + + + + Exposed building dependency modules/queryparser + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/package.html =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/package.html (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/package.html (revision ) @@ -0,0 +1,17 @@ + +Top-level package. Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java (revision ) @@ -0,0 +1,921 @@ +package org.apache.lucene.search.exposed.facet; + +import junit.framework.Test; +import junit.framework.TestSuite; +import junit.framework.TestCase; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.search.exposed.*; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; + +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.util.*; + +public class TestExposedFacets extends TestCase { + private ExposedHelper helper; + private ExposedCache cache; + + public TestExposedFacets(String name) { + super(name); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + cache = ExposedCache.getInstance(); + helper = new ExposedHelper(); + } + + @Override + + public void tearDown() throws Exception { + super.tearDown(); + cache.purgeAllCaches(); + helper.close(); + } + + + public static Test suite() { + return new TestSuite(TestExposedFacets.class); + } + + public static final String SIMPLE_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + +// " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public void testSimpleFacet() throws Exception { + final int DOCCOUNT = 1000; // Try with 5 + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a", "b"); + + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); +// new MockAnalyzer(new Random()new Random())); + Query q = qp.parse("true"); + searcher.search(q, TopScoreDocCollector.create(10, false)); + long preMem = getMem(); + long facetStructureTime = System.currentTimeMillis(); + + CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2); + + FacetRequest request = FacetRequest.parseXML(SIMPLE_REQUEST); + CollectorPool collectorPool = poolFactory.acquire(reader, request); + facetStructureTime = System.currentTimeMillis() - facetStructureTime; + + TagCollector collector; + FacetResponse response = null; + String sQuery = request.getQuery(); + for (int i = 0 ; i < 5 ; i++) { + collector = collectorPool.acquire(sQuery); + long countStart = System.currentTimeMillis(); + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); +// collector.collectAllValid(reader); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + response.setTotalTime(totalTime); +/* System.out.println("Collection #" + i + " for " + DOCCOUNT + + " documents in " + + getTime(System.currentTimeMillis()-countStart));*/ + collectorPool.release(sQuery, collector); + } + System.out.println("Document count = " + DOCCOUNT); + System.out.println("Facet startup time = " + getTime(facetStructureTime)); + System.out.println("Mem usage: preFacet=" + preMem + + " MB, postFacet=" + getMem() + " MB"); + if (response != null) { + System.out.println(response.toXML()); + } + searcher.close(); + reader.close(); + } + + private MockAnalyzer getAnalyzer() { + return new MockAnalyzer(new Random(), MockTokenizer.WHITESPACE, false); + } + + public static final String GROUP_REQUEST_ABC = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public static final String GROUP_ABC_EXPECTED = // TODO: Avoid hardcoding + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " "; + public void testMultiFacet() throws Exception { + final int DOCCOUNT = 6; + FacetRequest request = FacetRequest.parseXML(GROUP_REQUEST_ABC); + FacetResponse response = testMultiFacetHelper(request, DOCCOUNT); + //System.out.println(response.toXML()); + assertTrue("The response should contain \n" + GROUP_ABC_EXPECTED + + ", but was\n" + response.toXML(), + response.toXML().contains(GROUP_ABC_EXPECTED)); + } + + public static final String MINCOUNT_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public void testMinCountFacet() throws Exception { + final int DOCCOUNT = 6; + FacetRequest request = FacetRequest.parseXML(MINCOUNT_REQUEST); +// System.out.println("Requesting\n" + request.toXML()); + + FacetResponse response = testMultiFacetHelper(request, DOCCOUNT); + + assertFalse("The result should not contain any 'count=\"1\"' but was\n" + + response.toXML(), response.toXML().contains("count=\"1\"")); + assertTrue("The result should contain a 'count=\"2\"' but had\n" + + response.toXML(), response.toXML().contains("count=\"2\"")); + assertTrue("The result should contain a 'count=\"3\"' but had\n" + + response.toXML(), response.toXML().contains("count=\"3\"")); +// System.out.println(response.toXML()); + + } + + public static final String MINCOUNT_PREFIX_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public void testMinCountPrefixFacet() throws Exception { + final int DOCCOUNT = 6; + FacetRequest request = FacetRequest.parseXML(MINCOUNT_PREFIX_REQUEST); + System.out.println("Requesting\n" + request.toXML()); + + FacetResponse response = testMultiFacetHelper(request, DOCCOUNT); + + assertTrue("The result should contain a 'count=\"1\"' but had\n" + + response.toXML(), response.toXML().contains("count=\"1\"")); +// System.out.println(response.toXML()); + + } + + private FacetResponse testMultiFacetHelper( + FacetRequest request, int doccount) throws Exception { + final int DOCCOUNT = 6; + ExposedHelper helper = new ExposedHelper(); + File location = helper.buildMultiFieldIndex(DOCCOUNT); + + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.ALL, + getAnalyzer()); + Query q = qp.parse(ExposedHelper.ALL); + + CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2); + + CollectorPool collectorPool = poolFactory.acquire(reader, request); + + TagCollector collector = collectorPool.acquire(null); + searcher.search(q, collector); + FacetResponse response = collector.extractResult(request); + collectorPool.release(null, collector); + + searcher.close(); + helper.close(); // Cleanup + return response; + } + + public static final String SOLR_COMPARISON_REQUEST = + "\n" + + "\n" + + " replacable\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + /* + This attempts to re-create the Solr test for faceting found at + https://issues.apache.org/jira/browse/SOLR-475?focusedCommentId=12650071&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_12650071 + as details are sparse in the JIRA issue, comparisons cannot be made directly. + */ + public void testSolrComparisonFacet() throws Exception { + final int DOCCOUNT = 50000; + ExposedSettings.priority = ExposedSettings.PRIORITY.memory; + + File LOCATION = new File("/home/te/projects/index50M"); + if (!LOCATION.exists()) { + System.err.println("No index at " + LOCATION + ". A test index with " + + DOCCOUNT + " documents will be build"); + helper.createFacetIndex(DOCCOUNT); + LOCATION = ExposedHelper.INDEX_LOCATION; + } + IndexReader reader = ExposedIOFactory.getReader(LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + System.out.println("Index = " + LOCATION.getAbsolutePath() + + " (" + reader.maxDoc() + " documents)\n"); + + String FIELD = "hits10000"; + + QueryParser qp = new QueryParser( + Version.LUCENE_31, FIELD, + getAnalyzer()); + Query q = qp.parse("true"); + searcher.search(q, TopScoreDocCollector.create(10, false)); + System.out.println("Used heap after loading index and performing a " + + "simple search: " + getMem() + " MB\n"); + long preMem = getMem(); + long facetStructureTime = System.currentTimeMillis(); + + CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2); + + FacetRequest request = FacetRequest.parseXML(SOLR_COMPARISON_REQUEST); + CollectorPool collectorPool = poolFactory.acquire(reader, request); + facetStructureTime = System.currentTimeMillis() - facetStructureTime; + +/* long bMem = getMem(); + ExposedCache.getInstance().transitiveReleaseCaches(true); + System.out.println("Releasing leaf caches freed " + + (getMem() - bMem) + " MB"); +*/ + + TagCollector collector; + FacetResponse response = null; + request.setQuery(FIELD + ":true"); + String sQuery = null;// request.getQuery(); No caching + for (int i = 0 ; i < 5 ; i++) { + long countStart = System.currentTimeMillis(); + collector = collectorPool.acquire(sQuery); + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + collectorPool.release(sQuery, collector); + response.setTotalTime(totalTime); + System.out.println("Facet collection #" + i + " for " + DOCCOUNT + + " documents in " + + getTime(System.currentTimeMillis()-countStart)); + Thread.sleep(50); // Real world simulation or cheating? + } + System.out.println("Document count = " + DOCCOUNT); + System.out.println("Facet startup time = " + getTime(facetStructureTime)); + long fMem = getMem(); + System.out.println("Mem usage: preFacet = " + preMem + + " MB, postFacet = " + fMem + " MB. " + + "Facet overhead (approximate) = " + (fMem - preMem) + " MB."); + System.out.println(CollectorPoolFactory.getLastFactory()); + if (response != null) { + System.out.println(response.toXML()); + } + } + + private void dumpStats() { + System.out.println(CollectorPoolFactory.getLastFactory()); + System.out.println(ExposedCache.getInstance()); + } + + public static final String LOOKUP_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public void testIndexLookup() throws Exception { + final int DOCCOUNT = 200000; + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a", "b"); + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + Query q = qp.parse("true"); + searcher.search(q, TopScoreDocCollector.create(10, false)); + System.out.println( + "Index opened and test-sort performed, commencing faceting..."); + long preMem = getMem(); + + CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2); + + FacetResponse response = null; + assertTrue("There must be at least 10 documents in the index", + DOCCOUNT >= 10); + final int span = DOCCOUNT / 10; + long firstTime = -1; + long subsequents = 0; + int subCount = 0; + long poolTime = -1; + for (char prefix = 'A' ; prefix < 'X' ; prefix++) { + for (int offset = -span ; offset < span ; offset += span / 2) { + FacetRequest request = FacetRequest.parseXML( + LOOKUP_REQUEST.replace("myprefix", Character.toString(prefix)). + replace("myoffset", Integer.toString(offset))); + if (firstTime == -1) { + poolTime = System.currentTimeMillis(); + } + CollectorPool collectorPool = poolFactory.acquire(reader, request); + if (firstTime == -1) { + poolTime = System.currentTimeMillis() - poolTime; + System.out.println("Facet structure build finished in " + + getTime(poolTime) + ". Running requests...\n"); + } + + long countStart = System.currentTimeMillis(); + String sQuery = request.getQuery(); + TagCollector collector = collectorPool.acquire(sQuery); + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + assertNotNull("Extracting XML response should work", response.toXML()); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + if (firstTime == -1) { + firstTime = totalTime; + } else { + subCount++; + subsequents += totalTime; + } + response.setTotalTime(totalTime); +/* System.out.println("Collection for prefix " + prefix + " and offset " + + offset + " for " + DOCCOUNT + " documents took " + + (System.currentTimeMillis()-countStart) + "ms");*/ + collectorPool.release(sQuery, collector); + } + } + System.out.println("Document count = " + DOCCOUNT); + System.out.println("Facet structure build time = " + getTime(poolTime)); + System.out.println("First facet call = " + getTime(firstTime)); + System.out.println("Average of " + subCount + " subsequent calls = " + + getTime(subsequents / subCount)); + System.out.println("Mem usage: preFacet=" + preMem + + " MB, postFacet=" + getMem() + " MB"); + if (response != null) { + System.out.println(response.toXML()); + } + + } + + public void testBasicSearch() throws IOException, ParseException { + + File LOCATION = new File("/home/te/projects/index1M"); + if (!LOCATION.exists()) { + final int DOCCOUNT = 20000; + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a", "b"); + System.err.println("No index at " + LOCATION + ". A test index with " + + DOCCOUNT + " documents will be build at " + + ExposedHelper.INDEX_LOCATION ); + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + LOCATION = ExposedHelper.INDEX_LOCATION; + } + + final String field = "facet"; + final String term = "A"; + + assertTrue("No index at " + LOCATION.getAbsolutePath() + + ". Please build a test index (you can use one from on of the other" + + " JUnit tests in TestExposedFacets) and correct the path", + LOCATION.exists()); + + IndexReader reader = ExposedIOFactory.getReader(LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, field, + getAnalyzer()); + Query q = qp.parse(term); + System.out.println(q.toString()); + String sQuery = field + ":" + term; + + TopDocsCollector collector = TopScoreDocCollector.create(10, false); + searcher.search(q, collector); + assertTrue("There should be some hits for '" + sQuery + "'", + collector.getTotalHits() >0); + + } + + // MB + private long getMem() { + System.gc(); + return (Runtime.getRuntime().totalMemory() + - Runtime.getRuntime().freeMemory()) / 1048576; + } + + public static final String SCALE_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + /* + Performs sorted search, faceting and index lookup + */ + public void testScale() throws Exception { + long totalTime = -System.currentTimeMillis(); + + ExposedSettings.priority = ExposedSettings.PRIORITY.memory; + + File LOCATION = new File("/home/te/projects/index1M"); + if (!LOCATION.exists()) { + final int DOCCOUNT = 100000; + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a", "b"); + System.err.println("No index at " + LOCATION + ". A test index with " + + DOCCOUNT + " documents will be build at " + + ExposedHelper.INDEX_LOCATION ); + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + LOCATION = ExposedHelper.INDEX_LOCATION; + } + assertTrue("No index at " + LOCATION.getAbsolutePath() + + ". Please build a test index (you can use one from on of the other" + + " JUnit tests in TestExposedFacets) and correct the path", + LOCATION.exists()); + + IndexReader reader = ExposedIOFactory.getReader(LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + Query q = qp.parse("true"); + String sQuery = "even:true"; + + StringWriter sw = new StringWriter(10000); + sw.append("Index = " + LOCATION.getAbsolutePath() + " (" + reader.maxDoc() + + " documents)\n"); + + searcher.search(q, TopScoreDocCollector.create(10, false)); + sw.append("Used heap after loading index and performing a simple search: " + + getMem() + " MB\n"); + sw.append("Maximum possible memory (Runtime.getRuntime().maxMemory()): " + + Runtime.getRuntime().maxMemory() / 1048576 + " MB\n"); + System.out.println("Index " + LOCATION + + " opened and test search performed successfully"); + + sw.append("\n"); + + // Tests + CollectorPoolFactory poolFactory = new CollectorPoolFactory(6, 4, 2); + + qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + q = qp.parse("true"); + sQuery = "even:true"; + testScale(poolFactory, searcher, q, sQuery, sw); + +/* TODO: Check if this disabling works with the tests without re-computing + dumpStats(); + System.out.println("Disabling re-open optimization"); + ExposedCache.getInstance().transitiveReleaseCaches(true); + dumpStats(); + */ + + qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.MULTI, + getAnalyzer()); + q = qp.parse("A"); + sQuery = "multi:A"; // Strictly it's "facet:A", but that is too confusing + testScale(poolFactory, searcher, q, sQuery, sw); + + + System.out.println("**************************\n"); + + sw.append("\nUsed memory with sort, facet and index lookup structures " + + "intact: " + getMem() + " MB\n"); + totalTime += System.currentTimeMillis(); + sw.append("Total test time: " + getTime(totalTime)); + + System.out.println(""); + System.out.println(sw.toString()); + } + + private void testScale(CollectorPoolFactory poolFactory, + IndexSearcher searcher, Query query, String sQuery, StringWriter result) + throws IOException, XMLStreamException { + System.out.println("- Testing sorted search for " + sQuery); + testSortedSearch(searcher, "b", query, sQuery, null, result); + result.append("\n"); + + // Faceting + System.out.println("- Testing faceting for " + sQuery); + testFaceting(poolFactory, searcher, query, sQuery, result); + + // Index lookup + System.out.println("- Testing index lookup for " + sQuery); + TestIndexLookup(poolFactory, searcher, query, sQuery, result); + + } + + public static final String FACET_SCALE_SIMPLE_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public void testFacetScale() + throws XMLStreamException, IOException, ParseException { + + //CodecProvider.setDefaultCodec("Standard"); + + long totalTime = -System.currentTimeMillis(); + ExposedSettings.debug = true; + ExposedSettings.priority = ExposedSettings.PRIORITY.memory; + + //new ExposedHelper().close(); // Deletes old index + File LOCATION = ExposedHelper.INDEX_LOCATION; + if (!LOCATION.exists()) { + final int DOCCOUNT = 10000000; + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a"); + System.err.println("No index at " + LOCATION + ". A test index with " + + DOCCOUNT + " documents will be build at " + + ExposedHelper.INDEX_LOCATION ); + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + LOCATION = ExposedHelper.INDEX_LOCATION; + } + assertTrue("No index at " + LOCATION.getAbsolutePath() + + ". Please build a test index (you can use one from on of the other" + + " JUnit tests in TestExposedFacets) and correct the path", + LOCATION.exists() && LOCATION.listFiles().length > 0); + + IndexReader reader = ExposedIOFactory.getReader(LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + Query q = qp.parse("true"); + + StringWriter sw = new StringWriter(10000); + sw.append("Index = " + LOCATION.getAbsolutePath() + " (" + reader.maxDoc() + + " documents)\n"); + + searcher.search(q, TopScoreDocCollector.create(10, false)); + sw.append("Used heap after loading index and performing a simple search: " + + getMem() + " MB\n"); + + sw.append("\n"); + + // Tests + CollectorPoolFactory poolFactory = new CollectorPoolFactory(6, 4, 2); + + qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + q = qp.parse("true"); + String sQuery = "even:true"; + + System.out.println("- Testing faceting for " + sQuery); + testFaceting( + poolFactory, searcher, q, sQuery, sw, FACET_SCALE_SIMPLE_REQUEST); + System.out.println(sw.toString()); + System.out.println("Instances: " + ExposedTuple.instances); + } + +/* public void testP() { + final int COUNT = 60000000; + ArrayList tuples = new ArrayList(COUNT / 100); + long tupleTime = -System.currentTimeMillis(); + BytesRef b = new BytesRef("ss"); + for (int i = 0 ; i < COUNT ; i++) { + b.copy("ss"); + tuples.add(new ExposedTuple("foo", b, 0, 0)); + if (i % (COUNT / 100) == 0) { + tuples.clear(); + System.out.print("."); + } + } + tupleTime += System.currentTimeMillis(); + System.out.println("\nGot " + ExposedUtil.time("tuples", COUNT, tupleTime)); + tuples.clear(); + } + */ + private void testFaceting( + CollectorPoolFactory poolFactory, IndexSearcher searcher, Query q, + String sQuery, StringWriter result) + throws XMLStreamException, IOException { + testFaceting(poolFactory, searcher, q, sQuery, result, SCALE_REQUEST); + } + private void testFaceting( + CollectorPoolFactory poolFactory, IndexSearcher searcher, Query q, + String sQuery, StringWriter result, String requestString) + throws XMLStreamException, IOException { + IndexReader reader = searcher.getIndexReader(); + long facetStructureTime = -System.currentTimeMillis(); + + FacetResponse response = null; + int DOCCOUNT = reader.maxDoc(); + assertTrue("There must be at least 10 documents in the index", + DOCCOUNT >= 10); + final int span = DOCCOUNT / 10; + long firstTime = -1; + long subsequents = 0; + int subCount = 0; + long poolTime = -1; + for (int i = 0 ; i < 5 ; i++) { + FacetRequest request = FacetRequest.parseXML(requestString); + request.setQuery(sQuery); + if (firstTime == -1) { + poolTime = System.currentTimeMillis(); + } + CollectorPool collectorPool = poolFactory.acquire(reader, request); + if (firstTime == -1) { + poolTime = System.currentTimeMillis() - poolTime; + result.append("Facet pool acquisition for " + + "for \"" + sQuery + "\" with structure " + request.getGroupKey() + + ": " + getTime(poolTime) + "\n"); + } + + long countStart = System.currentTimeMillis(); + TagCollector collector = collectorPool.acquire(null); // No caching + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + assertNotNull("Extracting XML response should work", response.toXML()); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + if (firstTime == -1) { + firstTime = totalTime; + result.append("First faceting for " + sQuery + ": " + + getTime(totalTime) + "\n"); + } else { + subCount++; + subsequents += totalTime; + } + response.setTotalTime(totalTime); +/* System.out.println("Collection for prefix " + prefix + " and offset " + + offset + " for " + DOCCOUNT + " documents took " + + (System.currentTimeMillis()-countStart) + "ms");*/ + collectorPool.release(null, collector); // No caching + } + result.append("Subsequent " + subCount + " faceting calls (count caching " + + "disabled) response times: " + getTime(subsequents / subCount) + "\n"); + assertNotNull("There should be a response", response); + result.append(response.toXML()).append("\n"); + } + + private void TestIndexLookup( + CollectorPoolFactory poolFactory, IndexSearcher searcher, Query q, + String sQuery, StringWriter result) + throws XMLStreamException, IOException { + IndexReader reader = searcher.getIndexReader(); + long facetStructureTime = -System.currentTimeMillis(); + + FacetResponse response = null; + int DOCCOUNT = reader.maxDoc(); + assertTrue("There must be at least 10 documents in the index", + DOCCOUNT >= 10); + final int span = DOCCOUNT / 10; + long firstTime = -1; + long subsequents = 0; + int subCount = 0; + long poolTime = -1; + for (char prefix = 'A' ; prefix < 'X' ; prefix++) { + for (int offset = -span ; offset < span ; offset += span / 2) { + FacetRequest request = FacetRequest.parseXML( + LOOKUP_REQUEST.replace("myprefix", Character.toString(prefix)). + replace("myoffset", Integer.toString(offset))); + request.setQuery(sQuery); + if (firstTime == -1) { + poolTime = System.currentTimeMillis(); + } + CollectorPool collectorPool = poolFactory.acquire(reader, request); + if (firstTime == -1) { + poolTime = System.currentTimeMillis() - poolTime; + result.append("Initial lookup pool request (might result in structure" + + " building): " + getTime(poolTime) + "\n"); + } + + long countStart = System.currentTimeMillis(); + TagCollector collector = collectorPool.acquire(sQuery); + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + assertNotNull("Extracting XML response should work", response.toXML()); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + if (firstTime == -1) { + firstTime = totalTime; + result.append("First index lookup for \"" + sQuery + "\": " + + getTime(firstTime) + "\n"); + } else { + subCount++; + subsequents += totalTime; + } + response.setTotalTime(totalTime); +/* System.out.println("Collection for prefix " + prefix + " and offset " + + offset + " for " + DOCCOUNT + " documents took " + + (System.currentTimeMillis()-countStart) + "ms");*/ + collectorPool.release(sQuery, collector); + } + } + result.append("Subsequent " + subCount + " index lookups average " + + "response times: " + getTime(subsequents / subCount) + "\n"); + result.append(response.toXML()).append("\n"); + } + + public void testDumpNaturalSortedSearch() throws IOException, ParseException { + + File LOCATION = new File("/home/te/projects/index10M"); + if (!LOCATION.exists()) { + final int DOCCOUNT = 20000; + final int TERM_LENGTH = 20; + final int MIN_SEGMENTS = 2; + final List FIELDS = Arrays.asList("a", "b"); + System.err.println("No index at " + LOCATION + ". A test index with " + + DOCCOUNT + " documents will be build at " + + ExposedHelper.INDEX_LOCATION ); + helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS); + LOCATION = ExposedHelper.INDEX_LOCATION; + } + StringWriter sw = new StringWriter(); + IndexReader reader = ExposedIOFactory.getReader(LOCATION); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.EVEN, + getAnalyzer()); + Query q = qp.parse("true"); + String sQuery = "even:true"; + testSortedSearch(searcher, "a", q, sQuery, null, sw); + System.out.println(sw); + } + + private void testSortedSearch(IndexSearcher searcher, String field, Query q, + String sQuery, Locale locale, + StringWriter result) throws IOException { + // Sorted search + long firstSearch = -System.currentTimeMillis(); + ExposedFieldComparatorSource exposedFCS = + new ExposedFieldComparatorSource(searcher.getIndexReader(), locale); + Sort sort = new Sort(new SortField(field, exposedFCS)); + TopFieldDocs docs = searcher.search(q, null, 5, sort); + firstSearch += System.currentTimeMillis(); + + result.append("First natural order sorted search for \""+ sQuery + + "\" with " + docs.totalHits + " hits: " + + getTime(firstSearch) + "\n"); + + long subSearchMS = 0; + final int RUNS = 5; + final int MAXHITS = 5; + TopFieldDocs topDocs = null; + for (int i = 0 ; i < RUNS ; i++) { + subSearchMS -= System.currentTimeMillis(); + topDocs = searcher.search(q, MAXHITS, sort); + subSearchMS += System.currentTimeMillis(); + } + + result.append("Subsequent " + RUNS + + " sorted searches average response time: " + + getTime(subSearchMS / RUNS) + "\n"); + for (int i = 0 ; i < Math.min(topDocs.totalHits, MAXHITS) ; i++) { + int docID = topDocs.scoreDocs[i].doc; + result.append(String.format( + "Hit #%d was doc #%d with field " + field + " %s\n", + i, docID, + ((BytesRef)((FieldDoc)topDocs.scoreDocs[i]).fields[0]). + utf8ToString())); + } + } + + private String getTime(long ms) { + if (ms < 2999) { + return ms + " ms"; + } + long seconds = Math.round(ms / 1000.0); + long minutes = seconds / 60; + return String.format("%d:%02d minutes", minutes, seconds - (minutes * 60)); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/GrowingMutable.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/GrowingMutable.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/GrowingMutable.java (revision ) @@ -0,0 +1,210 @@ +package org.apache.lucene.util.packed; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A PackedInts that grows to accommodate any legal data that is added. + * Deltas for both array length and values are used so memory-representation + * is fairly compact. + *

+ * The initial default value for a growing mutable is the given minValue or 0 + * if no minValue is specified. If a value smaller than minValue is added after + * construction, the new default value will be that value and the old non-set + * values will still be the old default value. To put it in code: {@code + GrowingMutable g = new GrowingMutable(0, 5, 5, 10); + assert(g.get(0) == 5); + g.set(2, 4); + assert(g.get(0) == 5); + assert(g.get(1) == 4); + }. To avoid error it is imperative to either specify a minValue that is + guaranteed to be the minimum value for the life of the mutable or to ensure + that all values are assigned explicitly. + *

+ * The penalty for using this implementation compared to a plain PackedInts is + * slower {@link #set}s as it needs to check boundaries and potentially expand + * the internal array. There are no conditionals and very little extra overhead + * for {@link #get}s so access is still fast. + */ +// TODO: Optimize the speed of array length growth +public class GrowingMutable implements PackedInts.Mutable { + // Grow array length with this factor beyond the immediate need + private static final double GROWTH_FACTOR = 1.2; + + private int indexMin = 0; + private int indexMax = 0; + private long valueMin = 0; + private long valueMax = 0; + private int size = 0; // Size is explicit to handle grow seamlessly + private boolean optimizeSpeed = false; + + private PackedInts.Mutable values; + + /** + * Create an empty array of size 0. If it is possible to estimate the amount + * and rages of the values to add to the array, it is recommended to use + * {@link #GrowingMutable(int,int,long,long,boolean)} instead. + */ + public GrowingMutable() { + values = PackedInts.getMutable(0, 1); + } + + /** + * Create an empty array with the given boundaries. The boundaries will be + * expanded as needed. The default value is valueMin. + * @param indexMin the minimum index for the array. + * @param indexMax the maximum index for the array. + * @param valueMin the minimum value for the array. +* Note that this can be negative. + * @param valueMax the maximum value for the array. This must be higher than +* or equal to valueMin. + * @param optimizeSpeed if true, the number of bit required to represent the + * values are rounded to 1, 2, 3, 4, 8, 16, 32-40 or 64. + */ + public GrowingMutable(int indexMin, int indexMax, + long valueMin, long valueMax, boolean optimizeSpeed) { + if (indexMin > indexMax) { + throw new IllegalArgumentException(String.format( + "indexMin(%d) must be <= indexMax(%d)", indexMin, indexMax)); + } + if (valueMin > valueMax) { + throw new IllegalArgumentException(String.format( + "valueMin(%d) must be <= valueMax(%d)", valueMin, valueMax)); + } + this.indexMin = indexMin; + this.indexMax = indexMax; + this.valueMin = valueMin; + this.valueMax = valueMax; + this.size = indexMax - indexMin + 1; + this.optimizeSpeed = optimizeSpeed; + values = getMutable( + indexMax - indexMin + 1, PackedInts.bitsRequired(valueMax - valueMin)); + } + + private PackedInts.Mutable getMutable(int valueCount, int bitsPerValue) { + if (!optimizeSpeed) { + return PackedInts.getMutable(valueCount, bitsPerValue); + } + if (bitsPerValue <= 4 || + (bitsPerValue > 32 && bitsPerValue < 40)) { // TODO: Consider the 40 + return PackedInts.getMutable(valueCount, bitsPerValue); + } + return PackedInts.getMutable(valueCount, bitsPerValue); + } + + // Make sure that there is room. Note: This might adjust the boundaries. + private synchronized void checkValues(final int index, final long value) { + if (index >= indexMin && index <= indexMax + && value >= valueMin && value <= valueMax) { + return; + } + // TODO: Just adjust max is within bitsPerValue + if (size() == 0) { // First addition + indexMin = index; + indexMax = index; + valueMin = value; + valueMax = value; + GrowingMutable newGrowingMutable = new GrowingMutable( + indexMin, indexMax, valueMin, valueMax, optimizeSpeed); + size = 1; + values = newGrowingMutable.values; + return; + } + + // indexes are grown with factor, values without + int newIndexMin = + index >= indexMin ? indexMin : + indexMax - (int)Math.ceil((indexMax - index) * GROWTH_FACTOR); + int newIndexMax = + index <= indexMax ? indexMax : + indexMin + (int)Math.ceil((index - indexMin) * GROWTH_FACTOR); + long newValueMin = Math.min(value, valueMin); + long newValueMax = Math.max(value, valueMax); + + GrowingMutable newGrowingMutable = new GrowingMutable( + newIndexMin, newIndexMax, newValueMin, newValueMax, optimizeSpeed); + for (int i = indexMin ; i <= indexMax ; i++) { + newGrowingMutable.set(i, get(i)); + } + + values = newGrowingMutable.values; + indexMin = newIndexMin; + indexMax = newIndexMax; + valueMin = newValueMin; + valueMax = newValueMax; + size = indexMax - indexMin; // Size is always minimum to accommodate index + } + + public void set(int index, long value) { + checkValues(index, value); + values.set(index - indexMin, value - valueMin); + } + + @Override + public String toString() { + final int DUMP_SIZE = 10; + StringBuffer sb = new StringBuffer(DUMP_SIZE * 10); + sb.append("GrowingMutable("); + for (int i = indexMin ; i < indexMin + Math.min(size(), DUMP_SIZE) ; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append("i(").append(Integer.toString(i)).append(")="); + sb.append(Long.toString(get(i))); + } + if (size > DUMP_SIZE) { + sb.append(", ..."); + } + sb.append(")"); + return sb.toString(); + } + + /** + * Clear only resets the array. It does not free memory or adjust size. + */ + public void clear() { + values.clear(); + } + + public long get(int index) { + return values.get(index - indexMin) + valueMin; + } + + public int getBitsPerValue() { + return values.getBitsPerValue(); + } + + public int size() { + return size; + } + + public int getIndexMin() { + return indexMin; + } + + public int getIndexMax() { + return indexMax; + } + + public long getValueMin() { + return valueMin; + } + + public long getValueMax() { + return valueMax; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java (revision ) @@ -0,0 +1,228 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import com.ibm.icu.text.RawCollationKey; +import com.ibm.icu.text.Collator; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +/** + * Multi source Iterator that merges the ExposedTuples according to the given + * comparator. Designed to work tightly with {@link GroupTermProvider}. + *

+ * It is recommended to use {@link CachedTermProvider} with look-ahead on the + * sources or otherwise ensure that ordered iteration is fast if no docIDs are + * to be returned. If docIDs are to be returned, caching at the source level + * will not help. + *

+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless + * {@link #setReuseTuple(boolean)} is called with false. + *

+ * The merger works by keeping a list of the first elements from all sources + * and seeding a priority queue with indexes 0...#sources-1. When the PQ is + * popped, the elements corresponding to the indexes in the PQ are compared. + * When an element is returned, it is delivered to the iterator. The next time + * next() is called on the iterator, the index for the previous tuple is + * re-inserted into the PQ, unless the source for that index is empty. + * This ensures full reuse of the tuples and thus low impact on GC. + */ +class MergingTermDocIterator implements Iterator { + private final List> iterators; + private final boolean collectDocIDs; + private final GroupTermProvider groupProvider; + + private final ExposedTuple[] backingTuples; + private final ExposedPriorityQueue pq; + + private final Collator collator; + private final RawCollationKey[] backingKeys; // Only used with Collator + + // TODO: Make reuseTuple = true work + private boolean reuseTuple = false; + private long indirect = 0; + + /* + If pending is true, the currentIndex points to the iterator and backingTuple + that is to be drained. + */ + private boolean pending = false; // true == Terms or docIDs are ready in tuple + /** + * If pending is true, this is ready for delivery. + */ + private final ExposedTuple tuple = + new ExposedTuple("", new BytesRef(), -1, -1, null, -1); + /** + * If pending is true, this is the index for the source responsible for + * delivering the tuple. + */ + private int currentIndex = -1; + + /** + * If this is true and the given comparator is a + * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized + * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the + * chunk-sort speed. + */ + public static boolean optimizeCollator = true; + + public MergingTermDocIterator( + GroupTermProvider groupProvider, List sources, + Comparator comparator, boolean collectDocIDs) + throws IOException { + this.groupProvider = groupProvider; + this.collectDocIDs = collectDocIDs; + + iterators = new ArrayList>(sources.size()); + backingTuples = new ExposedTuple[sources.size()]; + + collator = optimizeCollator && + comparator instanceof ExposedComparators.BytesRefWrappedCollator + ? ((ExposedComparators.BytesRefWrappedCollator)comparator).getCollator() + : null; + backingKeys = new RawCollationKey[sources.size()]; + + ExposedComparators.OrdinalComparator wrappedComparator = + collator == null ? + ExposedComparators.wrapBacking(backingTuples, comparator) + : ExposedComparators.wrapBacking(backingKeys); + pq = new ExposedPriorityQueue(wrappedComparator, sources.size()); + int index = 0; + for (TermProvider source: sources) { + // TODO: Divide cacheLimit and avoid cache on collectDocIDs(?) + Iterator iterator = source.getIterator(collectDocIDs); + iterators.add(iterator); + ExposedTuple tuple = getNextTuple(index); + if (tuple != null) { + backingTuples[index] = tuple; + if (collator != null) { + RawCollationKey key = new RawCollationKey(); + backingKeys[index] = + collator.getRawCollationKey(tuple.term.utf8ToString(), key); + } + pq.add(index); + } + index++; + } + } + + // Ensure that we can deliver an id + public boolean hasNext() { + while (true) { + if (pending) { +// System.out.println("hasnext ***" + tuple.term.utf8ToString()); + return true; + } + if (pq.size() == 0) { + return false; + } + try { + seekToNextTuple(); + } catch (IOException e) { + throw new RuntimeException( + "IOException while seeking to next docID", e); + } + } + } + + public ExposedTuple next() { +// System.out.println("next " + tuple.term.utf8ToString()); + if (!hasNext()) { + throw new IllegalStateException("The iterator is depleted"); + } + ExposedTuple delivery = reuseTuple ? tuple : new ExposedTuple(tuple); + pending = false; +// System.out.println("Merging delivery: " + delivery); + return delivery; + } + + // Pops a tuple from the iterator indicated by the index and modifies term + // ordinal and docIDBase from segment to index level. + // If index == -1 or hasNext() == false, null is returned. + private ExposedTuple getNextTuple(int index) throws IOException { + if (index == -1 || !iterators.get(index).hasNext()) { + return null; + } + // TODO: Avoid the creation of a new tuple by assigning to existing tuple + ExposedTuple nextTuple = new ExposedTuple(iterators.get(index).next()); + long s = nextTuple.docIDBase; + nextTuple.docIDBase = (int) groupProvider.segmentToIndexDocID( + index, nextTuple.docIDBase); + nextTuple.ordinal = groupProvider.segmentToIndexTermOrdinal( + index, nextTuple.ordinal); + nextTuple.indirect = -1; // Just for good measure: It will be updated later + return nextTuple; + } + + private void seekToNextTuple() throws IOException { + +/* + if (collectDocIDs && currentIndex != -1) { + + // Start by checking if the previous source has more tuples with the + // same ordinal (e.g. if it has more docIDs) + ExposedTuple candidate = backingTuples[currentIndex]; + if (candidate != null && candidate.ordinal == tuple.ordinal) { + // Field, term, ordinal and indirect are the same + tuple.docID = candidate.docID; + pending = true; + backingTuples[currentIndex] = getNextTuple(currentIndex); + return; + } + } + */ + // DocID-optimization must take place before reinserting into pq + + // Take the next tuple + while (!pending && pq.size() > 0) { + final ExposedTuple newTuple = pop(); + if (newTuple == null) { + break; + } + + if (tuple.term.equals(newTuple.term)) { + if (!collectDocIDs) { + continue; // Skip duplicates when no docID + } + tuple.set(newTuple.field, tuple.term, tuple.ordinal, tuple.indirect, + newTuple.docIDs, newTuple.docIDBase); + } else { + tuple.set(newTuple.field, newTuple.term, newTuple.ordinal, indirect++, + newTuple.docIDs, newTuple.docIDBase); + } + pending = true; + } + } + + private ExposedTuple pop() throws IOException { + currentIndex = pq.pop(); + if (currentIndex == -1) { + return null; + } + final ExposedTuple foundTuple = backingTuples[currentIndex]; + + final ExposedTuple newTuple = getNextTuple(currentIndex); + if (newTuple != null) { + backingTuples[currentIndex] = newTuple; + if (collator != null) { + RawCollationKey key = new RawCollationKey(); + backingKeys[currentIndex] = + collator.getRawCollationKey(newTuple.term.utf8ToString(), key); + } + pq.add(currentIndex); + } + return foundTuple; + } + + public void remove() { + throw new UnsupportedOperationException("Not a valid operation"); + } + + public void setReuseTuple(boolean reuseTuple) { + this.reuseTuple = reuseTuple; + } + +} \ No newline at end of file Index: lucene/contrib/exposed/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java (revision ) @@ -0,0 +1,47 @@ +package org.apache.lucene.util.packed; + +import org.apache.lucene.util.LuceneTestCase; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class TestGrowingMutable extends LuceneTestCase { + + public void testMiscGrowth() throws Exception { + GrowingMutable g = new GrowingMutable(); + + g.set(100, 1000); + assertEquals("The value at index 100 should be correct", 1000, g.get(100)); + assertEquals("The size should be correct", 1, g.size()); + + g.set(98, 5000); + assertEquals("The value at index 98 should still be correct", + 5000, g.get(98)); + assertEquals("The value at index 100 should still be correct", + 1000, g.get(100)); + assertEquals("The size should be extended", 3, g.size()); + + g.set(-123, -12345); + assertEquals("The value at index 98 should still be correct", + 5000, g.get(98)); + assertEquals("The value at index 100 should still be correct", + 1000, g.get(100)); + assertEquals("The value at -123 should be as defined", + -12345, g.get(-123)); + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java (revision ) @@ -0,0 +1,136 @@ +package org.apache.lucene.search.exposed.facet; + +import java.util.*; + +/** + * Holds a number of {@link TagCollector}s tied to a single {@link FacetMap} + * and ensures that they are cleared and ready for use. If there are no + * collectors in the pool when a new one is acquired, a new collector will be + * created. + *

+ * The pool holds a mix of cleared and ready for use {@link TagCollector}s as + * well as updated collectors from previous queries. When a collector is + * released back into the pool it is stored in the non-cleared cache. If the + * non-cleared cache is full, the oldest entry is cleared and moved into the + * cleared cache. + *

+ * The pool is thread-safe and thread-efficient. + */ +// TODO: Consider making a limit on the number of collectors to create +public class CollectorPool { + private final List fresh; + private final Map filled; + private FacetMap map; + private final int freshCollectors; + private final int maxFilled; + + /** + * When a TagCollector is released with a query, it is stored as filled. If + * the cache for filled is full, the oldest entry is cleared and stored in + * the fresh pool. + * @param map the map to use the collectors with. + * @param filledCollectors the maximum number of previously filled collectors. + * @param freshCollectors the maximum number of fresh collectors. + */ + public CollectorPool( + FacetMap map, int filledCollectors, int freshCollectors) { + this.map = map; + this.freshCollectors = freshCollectors; + fresh = new ArrayList(); + this.maxFilled = filledCollectors; + filled = new LinkedHashMap(filledCollectors) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + if (size() > maxFilled) { + releaseFresh(remove(eldest.getKey())); // Send it to the fresh cache + } + return false; // We handle the removal ourselves + } + }; + } + + /** + * @param query the query associated with the collector. If the pool contains + * a collector filled from the query, it is returned. Else a cleared collector + * is returned. null is a valid query and will always return a cleared + * collector. + *

+ * Note: If a non-null query is provided, the caller must check the returned + * collector with the call {@link TagCollector#getQuery()}. If the result is + * not null, the collector was taken from cache and is already filled. In that + * case, the caller should not make a new search with the collector, but + * instead call {@link TagCollector#extractResult(org.apache.lucene.search.exposed.facet.request.FacetRequest)} directly. + * @return a recycled collector if one is available, else a new collector. + */ + public TagCollector acquire(String query) { + synchronized (filled) { + if (query != null && maxFilled != 0 && filled.containsKey(query)) { + return filled.remove(query); + } + } + + synchronized (fresh) { + for (int i = 0 ; i < fresh.size() ; i++) { + if (!fresh.get(i).isClearRunning()) { + return fresh.remove(i); + } + } + } + return new TagCollector(map); + } + + /** + * Releasing a collector sends it to the filled cache. If the filled cache is + * full, the oldest entry is cleared with a delayedClear and sent to the fresh + * cache. If the maximum capacity of the fresh cache has been reached, the + * collector is discarded. + *

+ * This method exits immediately. + * @param query the query used for filling this collector. If null, the + * collector is always going to the fresh cache. + * @param collector the collector to put back into the pool for later reuse. + */ + public void release(String query, TagCollector collector) { + synchronized (filled) { + if (maxFilled > 0 && query != null) { + collector.setQuery(query); + filled.put(query, collector); + return; + } + } + releaseFresh(collector); + } + + private void releaseFresh(TagCollector collector) { + synchronized (fresh) { + if (fresh.size() >= freshCollectors) { + return; + } + collector.delayedClear(); + fresh.add(collector); + } + } + + public void clear() { + synchronized (fresh) { + fresh.clear(); + } + synchronized (filled) { + filled.clear(); + } + } + + public String toString() { + long total = 0; + for (Map.Entry entry: filled.entrySet()) { + total += entry.getValue().getMemoryUsage(); + } + for (TagCollector collector: fresh) { + total += collector.getMemoryUsage(); + } + return "CollectorPool(" + map.toString() + ", #fresh counters = " + + fresh.size() + ", #filled counters = " + filled.size() + + ", total counter size = " + total / 1024 + " KB)"; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java (revision ) @@ -0,0 +1,281 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.search.exposed.facet.request.FacetRequestGroup; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.OpenBitSet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Counts tag occurrences in the given documents. This collector can be used as + * a standard Lucene collector or it can be filled by specifying the document + * IDs with an {@link OpenBitSet} or a standard {@code int[]}}. + */ +public class TagCollector extends Collector { + private String query = null; + private long countTime = -1; + private int docBase; + private final int[] tagCounts; + private final FacetMap map; + private boolean clearRunning = false; + private long hitCount = 0; +// TODO: Rememver query for caching of results + public TagCollector(FacetMap map) { + this.map = map; + this.tagCounts = new int[map.getTagCount()]; + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + //Ignore + } + + /* + Final is annoying, but we need all the speed we can get in the inner loop + */ + @Override + public final void collect(final int doc) throws IOException { + hitCount++; + map.updateCounter(tagCounts, doc + docBase); + } + +/* @Override + public void setNextReader(IndexReader reader, int docBase) throws IOException { + this.docBase = docBase; + }*/ + + @Override + public void setNextReader(IndexReader.AtomicReaderContext context) + throws IOException { + docBase = context.docBase; + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + /** + * Uses the given bits to update the tag counts. Each bit designates a docID. + *

+ * Note: This is different from calling {@link #collect(int)} repeatedly + * as the single docID collect method adjusts for docBase given in + * {@link #setNextReader}. + * @param bits the document IDs to use to use for tag counting. + * @throws java.io.IOException if the bits could not be accessed. + */ + public void collect(OpenBitSet bits) throws IOException { + countTime = System.currentTimeMillis(); + hitCount = 0; + DocIdSetIterator ids = bits.iterator(); + int id; + while ((id = ids.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + hitCount++; + map.updateCounter(tagCounts, id); + } + countTime = System.currentTimeMillis() - countTime; + } + + /** + * Uses the given docIDs to update the tag counts. + *

+ * Note: This is different from calling {@link #collect(int)} repeatedly + * as the single docID collect method adjusts for docBase given in + * {@link #setNextReader}. + * @param docIDs the document IDs to use for tag counting. + */ + public void collect(int[] docIDs) { + countTime = System.currentTimeMillis(); + hitCount = docIDs.length; + for (int docID: docIDs) { + map.updateCounter(tagCounts, docID); + } + countTime = System.currentTimeMillis() - countTime; + } + + /** + * Collect all docIDs except those marked as deleted. + * @param reader a reader for the index. + * @throws java.io.IOException if the bits could not be accessed. + */ + // TODO: Rewrite this to avoid allocation of the large bitset + public void collectAllValid(IndexReader reader) throws IOException { + countTime = System.currentTimeMillis(); + OpenBitSet bits = new OpenBitSet(reader.maxDoc()); + bits.flip(0, reader.maxDoc()); + if (reader.getSequentialSubReaders() == null) { + remove(bits, 0, reader.getLiveDocs()); + } else { + int docBase = 0; + for (IndexReader sub: reader.getSequentialSubReaders()) { + remove(bits, docBase, sub.getLiveDocs()); + // sub.getTopReaderContext().docBaseInParent + docBase += sub.maxDoc(); + } + } + collect(bits); + countTime = System.currentTimeMillis() - countTime; + } + private void remove(OpenBitSet prime, int base, Bits keepers) { + if (keepers == null) { + return; + } + // TODO: Optimize for OpenBitSet + for (int i = 0 ; i < keepers.length() ; i++) { + if (!keepers.get(i)) { + prime.clear(base + i); + } + } + } + + /** + * Clears the collector making it ready for reuse. It is highly recommended to + * reuse collectors as it lowers Garbage Collection impact, especially for + * large scale faceting. + *

+ * Consider using {@link #delayedClear()} to improve responsiveness. + */ + public void clear() { + clearRunning = true; + Arrays.fill(tagCounts, 0); + hitCount = 0; + query = null; + clearRunning = false; + } + + /** + * Starts a Thread that clears the collector and exits immediately. Make sure + * that {@link #isClearRunning()} returns false before using the collector + * again. + */ + public void delayedClear() { + new Thread(new Runnable() { + public void run() { + clear(); + } + }, "TagCollector clear").start(); + } + + public boolean isClearRunning() { + return clearRunning; + } + + public String toString() { + return "TagCollector(" + tagCounts.length + " potential tags from " + + map.toString(); + } + + public String toString(boolean verbose) { + if (!verbose) { + return toString(); + } + long nonZero = 0; + long sum = 0; + for (int count: tagCounts) { + if (count > 0) { + nonZero++; + } + sum += count; + } + return "TagCollector(" + tagCounts.length + " potential tags, " + nonZero + + " non-zero counts, total sum " + sum + " from " + map.toString(); + } + + /** + * After collection (and implicit reference counting), a response can be + * extracted. Note that countTime and totalTime is not set in the response. + * It is recommended that these two time measurements are updated before + * the response is delivered. + * @param request a request matching the one used for counting. Note that + * minor details, such as offset and maxTags need not match. + * @return a fully resolved facet structure. + * @throws IOException if the facet mapper could not deliver terms. + */ + public FacetResponse extractResult(FacetRequest request) throws IOException { + if (map.getIndirectStarts().length-1 != request.getGroups().size()) { + throw new IllegalStateException( + "The number of term providers in the FacetMap was " + + (map.getIndirectStarts().length-1) + + ", while the number of groups in the request was " + + request.getGroups().size() + ". The two numbers must match"); + } + List responseGroups = + new ArrayList(request.getGroups().size()); + // TODO: Consider threading larger facets, but beware of cache blowout + for (int i = 0 ; i < request.getGroups().size() ; i++) { + FacetRequestGroup requestGroup = request.getGroups().get(i); +// System.out.println("Extracting for " + requestGroup.getGroup().getName() + ": " + startTermPos + " -> " + endTermPos); + responseGroups.add(extractResult(requestGroup, i)); + } + FacetResponse response = + new FacetResponse(request, responseGroups, hitCount); + response.setCountingTime(getCountTime()); + return response; + } + + private FacetResponse.Group extractResult( + FacetRequestGroup requestGroup, int groupID) throws IOException { +/* if (!FacetRequest.ORDER_COUNT.equals(requestGroup.getOrder())) { + throw new UnsupportedOperationException("The order '" + + requestGroup.getOrder() + " is not supported yet for result " + + "extraction"); + } + */ + int startTermPos = map.getIndirectStarts()[groupID]; // Inclusive + int endTermPos = map.getIndirectStarts()[groupID+1]; // Exclusive + return new TagExtractor(requestGroup).extract( + groupID, map, tagCounts, startTermPos, endTermPos); + } + + + /** + * The query is typically used for cache purposes. + * @param query the query used for the collector filling search. + */ + public void setQuery(String query) { + this.query = query; + } + + /** + * @return the query used for filling this collector. Must be set explicitly + * by the user of this class with {@link #setQuery(String)}. + */ + public String getQuery() { + return query; + } + + public long getHitCount() { + return hitCount; + } + + /** + * This is set implicitely by calls to {@link #collect(int[])} and + * {@link #collect(OpenBitSet)} but must be set explicitly with + * {@link #setCountTime(long)} if the TagCollector is used as a Lucene + * collector with a standard search. + * @return the number of milliseconds used for filling the tag counter. + */ + public long getCountTime() { + return countTime; + } + + /** + * @param countTime the number of milliseconds used for filling the counter. + */ + public void setCountTime(long countTime) { + this.countTime = countTime; + } + + // Approximate + long getMemoryUsage() { + return tagCounts.length * 4; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java (revision ) @@ -0,0 +1,204 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.packed.GrowingMutable; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; + +/** + * Default implementation of some methods from TermProvider. + */ +public abstract class TermProviderImpl implements TermProvider { + protected boolean cacheTables; + private PackedInts.Mutable docToSingle = null; + + private final IndexReader reader; + private int docIDBase; // Changed on reopen + private final Comparator comparator; + private final String comparatorID; + private final String designation; + + /** + * @param reader the underlying reader for this provider. + * @param docIDBase the base to add to all docIDs returned from this provider. + * @param comparator the comparator used for sorting. If this is null, the + * natural BytesRef-order is used. + * @param comparatorID an ID for the comparator. If this is natural order, + * this should be {@link ExposedRequest#LUCENE_ORDER}. + * @param designation used for feedback and debugging. No formal requirements. + * @param cacheTables if true, tables such as orderedOrdinals, docID2indirect + * and similar should be cached for re-requests after generation. + */ + protected TermProviderImpl( + IndexReader reader, int docIDBase, + Comparator comparator, String comparatorID, + String designation, boolean cacheTables) { + this.reader = reader; + this.docIDBase = docIDBase; + this.comparator = comparator; + this.comparatorID = comparatorID; + this.cacheTables = cacheTables; + this.designation = designation; + } + + public synchronized PackedInts.Reader getDocToSingleIndirect() + throws IOException { + if (docToSingle != null) { + return docToSingle; + } + if (getMaxDoc() > Integer.MAX_VALUE) { + throw new UnsupportedOperationException( + "Unable to handle more than Integer.MAX_VALUE documents. " + + "Got macDocs " + getMaxDoc()); + } + // TODO: getMaxDoc() is seriously wonky for GroupTermProvider. What breaks? + long sortTime; + // TODO: Check why it is extremely slow to start with low maxValue + PackedInts.Mutable docToSingle = new GrowingMutable( + 0, (int)getMaxDoc(), -1, getOrdinalTermCount(), + ExposedSettings.priority == ExposedSettings.PRIORITY.speed); + try { + Iterator ei = getIterator(true); + sortTime = System.currentTimeMillis(); + while (ei.hasNext()) { + ExposedTuple tuple = ei.next(); + if (tuple.docIDs != null) { + int read; + final DocsEnum.BulkReadResult bulk = tuple.docIDs.getBulkResult(); + final IntsRef ints = bulk.docs; + while ((read = tuple.docIDs.read()) > 0) { + final int to = read + ints.offset; + for (int i = ints.offset ; i < to ; i++) { + docToSingle.set((int)(ints.ints[i] + tuple.docIDBase), + tuple.indirect); + } + } +/* + int doc; + // TODO: Test if bulk reading (which includes freqs) is faster + while ((doc = tuple.docIDs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { + docToSingle.set((int)(doc + tuple.docIDBase), tuple.indirect); + } */ + } +/* if (tuple.indirect << 60 == 0) { + System.out.print("."); + }*/ + } + } catch (IOException e) { + throw new RuntimeException("Unable to create doc to indirect map", e); + } + sortTime = System.currentTimeMillis() - sortTime; + if (ExposedSettings.debug) { + System.out.println(this.getClass().getSimpleName() + " merge-mapped " + + (getMaxDoc()-1) + " docs to single terms in " + sortTime + " ms: " + + (sortTime == 0 ? "N/A" : docToSingle.size() / sortTime) + " docs/ms"); + } + if (cacheTables) { + this.docToSingle = docToSingle; + } + return docToSingle; + } + + public String getDesignation() { + return designation; + } + + public Comparator getComparator() { + return comparator; + } + + public String getComparatorID() { + return comparatorID; + } + + public IndexReader getReader() { + return reader; + } + + public void setDocIDBase(int base) { + docIDBase = base; + } + + public int getDocIDBase() { + return docIDBase; + } + + public int getNearestTermIndirect(BytesRef key) throws IOException { + return getNearestTermIndirect(key, 0, (int)getUniqueTermCount()); + } + public int getNearestTermIndirect( + final BytesRef key, int startTermPos, int endTermPos) throws IOException { + if (getComparator() != null) { + return getNearestWithComparator(key, startTermPos, endTermPos); + } + int low = startTermPos; + int high = endTermPos-1; + while (low <= high) { + int middle = low + high >>> 1; + BytesRef midVal = getOrderedTerm(middle); + int compValue = midVal.compareTo(key); + if (compValue < 0) { + low = middle + 1; + } else if (compValue > 0) { + high = middle - 1; + } else { + return middle; + } + } + return low; + } + + private int getNearestWithComparator( + final BytesRef key, int startTermPos, int endTermPos) throws IOException { + final Comparator comparator = getComparator(); + int low = startTermPos; + int high = endTermPos-1; + while (low <= high) { + int middle = low + high >>> 1; + BytesRef midVal = getOrderedTerm(middle); + int compValue = comparator.compare(midVal, key); + if (compValue < 0) { + low = middle + 1; + } else if (compValue > 0) { + high = middle - 1; + } else { + return middle; + } + } + return low; + } + + protected String packedSize(PackedInts.Reader packed) { + long bytes = packed.size() * packed.getBitsPerValue() / 8; + if (bytes > 1048576) { + return bytes / 1048576 + " MB"; + } + if (bytes > 1024) { + return bytes / 1024 + " KB"; + } + return bytes + " bytes"; + } + + @Override + public String toString() { + if (docToSingle == null) { + return "TermProviderImpl(" + getDesignation() + + ", no docToSingle cached)"; + } + return "TermProviderImpl(" + getDesignation() + ", docToSingle.length=" + + docToSingle.size() + " mem=" + packedSize(docToSingle) + ")"; + } + + public void transitiveReleaseCaches(int level, boolean keepRoot) { + if (keepRoot && level == 0) { + return; + } + docToSingle = null; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java (revision ) @@ -0,0 +1,84 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; + +import java.io.IOException; + +/** + * ExposedFacets relies on the ordinal-oriented structures from the exposed + * framework. It shares structures with sorters whenever possible. + *

+ * The overall goal of exposed faceting is to minimize memory usage for + * faceting. This comes at a slight cost in processing speed as PackedInts + * are used to hold arrays of integers instead of int[]. The faceting system + * used delayed resolving of tags, meaning that only the returned tags are + * resolved from internal IDs to Strings. The resolve process taxes the + * IO-system and will likely be slow on a setup with a small disk cache, + * large index and spinning disks. For those not in-the-know, switching to + * SSDs for the index is the current silver bullet in the year 2010 for search + * responsiveness in general and works very well with the exposed framework. + *

+ * For each facet, a {@link org.apache.lucene.search.exposed.TermProvider} + * handles the ordinal to term mapping. + * This allows for locale based sorting of tags and similar. A supermap is + * created from document id to TermProvider entries by treating the TermProvider + * entry positions as logically increasing. Example: TermProvider A has 10 + * entries, provider B has 20 entries. When a document is mapped to entry 5 + * from provider B, the value 10 + 5 is stored in the map. + *

+ * The supermap is logically a two-dimensional integer array where + * supermap[docID] contains an array of TermProvider entry positions. + * As Java's implementation of multidimensional arrays is not memory efficient, + * this implementation uses indirection where one array designates the starting + * and ending entry in another array. + *

+ * The memory usage for 10M documents with 5M unique tags with 9M references in + * facet A, 15M unique tags with 15M references in facet B and 10 unique tags + * with 20M references in facet C is thus {@code +(9M + 15M + 20M) * log2(5M + 15M + 10) bits = for tag references +10M * log2(9M + 15M + 20M) bits for the doc2tagref array +5M * log2(5M) + 15M * log2(15M) + 10 * log2(10) bits TermProviders +} which gives a total of 220 MB. Depending on setup, sorting and the need for + * fast re-opening of the index, an extra copy of the TermProvider-data might + * be needed, bringing the total memory usage up to 275 MB. In order to actually + * use the structure for faceting, a counting array is needed. This is an int[] + * with an entry for each unique tag (20000010 or about 20M in this example). + * This adds an extra 80 MB for each concurrent thread. + *

+ * There is some temporary memory overhead while building the structures, but + * this is less than the total size of the final structures so a rule of thumb + * is to double that size to get the upper bound. Thus 550 MB of heap is + * required for the whole shebang, which leaves (550-275)/80MB ~= 3 concurrent + * faceting searches. + *

+ * Scaling down, 1M documents with a single facet with 500K unique tags and + * 1M occurrences takes about 20 MB with 3 concurrent searches. + *

+ * Scaling up, 100M documents with 3 facets with 100M unique tags and + * 1000M occurrences each can be done on a 24GB machine. However, counting + * the tags from a search result would take a while. + */ +public class ExposedFacets { + /** + * The maximum number of collectors to hold in the collector pool. + * Each collector takes up 4 * #documents bytes in the index. + */ + private static final int MAX_COLLECTORS = 10; + + private CollectorPool collectorPool; + + public void facet(FacetRequest request, IndexSearcher searcher, Query query) + throws IOException { + TagCollector collector = collectorPool.acquire(request.getQuery()); + try { + searcher.search(query, collector); + } finally { + collectorPool.release(request.getQuery(), collector); + } + } + + + // TODO: Support match-all without search +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java (revision ) @@ -0,0 +1,155 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.exposed.ExposedCache; +import org.apache.lucene.search.exposed.ExposedSettings; +import org.apache.lucene.search.exposed.TermProvider; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.search.exposed.facet.request.FacetRequestGroup; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Constructs {@link CollectorPool}s based on {@link org.apache.lucene.search.exposed.facet.request.FacetRequest}s. As the + * construction of a pool is costly, pools are kept in a LRU-cache. + *

+ * The factory connects to the {@link ExposedCache} and the cache is + * automatically updated (aka cleared) when the index is reloaded and IndexCache + * is notified about it. + */ +public class CollectorPoolFactory implements ExposedCache.PurgeCallback { + private Map poolMap; + /** + * The maximum number of filled collectors to cache for each CollectorPool. + */ + private final int filledCollectors; + + /** + * The maximum number of fresh collectors to cache for each CollectorPool. + */ + private final int freshCollectors; + + private static CollectorPoolFactory lastFactory = null; + + /** + * Constructing a CollectorPool is very costly, while constructing a collector + * in a collector pool only impacts garbage collections. In order to avoid + * excessive CPU-load, it is highly recommended to tweak maxSize, + * freshCollectors and filledCollectors so that maxSize is never reached. + *

+ * A new CollectorPool is constructed every time the group in the + * {@link org.apache.lucene.search.exposed.facet.request.FacetRequest} is modified. A new collector is constructed when + * a facet request is made and all current collectors for the given pool + * are active. + * @param maxSize the maximum number of CollectorPools to keep cached. + * @param filledCollectors the maximum number of fresh collectors in each + * pool. + * @param freshCollectors the maximum number of fresh collectors in each pool. + * @see {@link CollectorPool(FacetMap, int, int)}. + */ + public CollectorPoolFactory( + final int maxSize, int filledCollectors, int freshCollectors) { + poolMap = new LinkedHashMap(maxSize, 0.75f, true) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() > maxSize; + } + }; + this.freshCollectors = freshCollectors; + this.filledCollectors = filledCollectors; + lastFactory = this; + ExposedCache.getInstance().addRemoteCache(this); + } + + /** + * An ugly hack to get singleton-like behaviour with initialization. + * @return the last factory generated. + */ + // TODO: Replace this with proper singleton + public static CollectorPoolFactory getLastFactory() { + return lastFactory; + } + + /** + * If a matching CollectorPool exists, it is returned. If not, a new pool is + * created. Note that pool creation is costly. + * @param reader a reader for the full index. + * @param request specified the behaviour of the wanted CollectorPool. + * @return a CollectorPool matching the request. + * @throws java.io.IOException if the reader could not be accessed. + */ + public synchronized CollectorPool acquire( + IndexReader reader, FacetRequest request) throws IOException { + final String key = request.getGroupKey(); + CollectorPool pool = poolMap.get(key); + if (pool != null) { + poolMap.put(key, pool); // Re-insert to support LRU-pool + return pool; + } + + if (ExposedSettings.debug) { + System.out.println("Creating CollectorPool for " + key); + } + List groups = request.getGroups(); + List termProviders = + new ArrayList(groups.size()); + for (FacetRequestGroup group: groups) { + TermProvider provider = ExposedCache.getInstance().getProvider( + reader, group.getGroup()); + if (group.isHierarchical()) { + provider = new HierarchicalTermProvider( + provider, group.getDelimiter()); + } + termProviders.add(provider); + } + + long mapTime = -System.currentTimeMillis(); + FacetMap facetMap = new FacetMap(reader.maxDoc(), termProviders); + mapTime += System.currentTimeMillis(); +// System.out.println("Map time: " + mapTime + "ms"); + pool = new CollectorPool(facetMap, filledCollectors, freshCollectors); + poolMap.put(key, pool); + return pool; + } + + public synchronized void clear() { + poolMap.clear(); + } + + public void purgeAllCaches() { + if (ExposedSettings.debug) { + System.out.println("CollectorPoolFactory.purgeAllCaches() called"); + } + clear(); + } + + public void purge(IndexReader r) { + if (ExposedSettings.debug) { + System.out.println("CollectorPoolFactory.purge(" + r + ") called"); + } + // TODO: Make a selective clear + clear(); + } + + public String toString() { + StringWriter sw = new StringWriter(); + sw.append("CollectorPoolFactory("); + boolean first = true; + for (Map.Entry entry: poolMap.entrySet()) { + if (first) { + first = false; + } else { + sw.append(", "); + } + sw.append(entry.getValue().toString()); + } + sw.append(")"); + return sw.toString(); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedCache.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedCache.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedCache.java (revision ) @@ -0,0 +1,187 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.exposed.facet.request.FacetRequestGroup; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.*; + +public class ExposedCache { + + private final ArrayList cache = new ArrayList(5); + private final List remoteCaches = + new ArrayList(); + + private static final ExposedCache exposedCache; + static { + exposedCache = new ExposedCache(); + } + public static ExposedCache getInstance() { + return exposedCache; + } + + private ExposedCache() { + } + + public void addRemoteCache(PurgeCallback callback) { + remoteCaches.add(callback); + } + + public boolean removeRemoteCache(PurgeCallback callback) { + return remoteCaches.remove(callback); + } + + // NOTE: Reverse is ignored in providers + public TermProvider getProvider( + IndexReader reader, ExposedRequest.Group group) throws IOException { + return getProvider(reader, group.getName(), group.getFieldNames(), + group.getComparator(), group.getComparatorID()); + } + + public TermProvider getProvider( + IndexReader reader, String groupName, List fieldNames, + Comparator comparator,String comparatorID) + throws IOException { + ExposedRequest.Group groupRequest = FacetRequestGroup.createGroup( + groupName, fieldNames, comparator, false, comparatorID); + + for (TermProvider provider: cache) { + if (provider instanceof GroupTermProvider + && ((GroupTermProvider) provider).getRequest().worksfor(groupRequest) + && provider.getReaderHash() == reader.hashCode()) { + return provider; + } + } + if (ExposedSettings.debug) { + System.out.println("Creating provider for " + groupName); + } + + // No cached value. Modify the comparator IDs to LUCENE-order if they were + // stated as free. Aw we create the query ourselves, this is okay. + groupRequest.normalizeComparatorIDs(); + + boolean isSingle = true; + IndexReader[] readers; + if (reader.getSequentialSubReaders() == null) { + readers = new IndexReader[1]; + readers[0] = reader; + } else { + readers = reader.getSequentialSubReaders(); + isSingle = false; + } + + List fieldProviders = + new ArrayList(readers.length * fieldNames.size()); + + long fieldProviderConstruction = -System.currentTimeMillis(); + int docBase = 0; + for (IndexReader sub: readers) { + // TODO: Why is the docBase always 0? +/* int docBase = ((IndexReader.AtomicReaderContext)sub.getTopReaderContext()).docBase; + System.out.println("Skipping to reader of type " + sub.getClass().getSimpleName() + " with docBase=" + docBase + " and maxDoc=" + sub.maxDoc());*/ + for (ExposedRequest.Field fieldRequest: groupRequest.getFields()) { + fieldProviders.add(getProvider( + sub, isSingle ? 0 : docBase, fieldRequest, true, true)); + } + // Used in DirectoryReader.initialize so it should be fairly safe + docBase += sub.maxDoc(); + } + fieldProviderConstruction += System.currentTimeMillis(); + + long groupProviderconstruction = -System.currentTimeMillis(); + TermProvider groupProvider = new GroupTermProvider( + reader.hashCode(), fieldProviders, groupRequest, true); + groupProviderconstruction += System.currentTimeMillis(); + cache.add(groupProvider); + +// System.out.println("Field: " + fieldProviderConstruction +// + "ms, group: " + groupProviderconstruction + "ms"); + return groupProvider; + } + + FieldTermProvider getProvider( + IndexReader segmentReader, int docIDBase, ExposedRequest.Field request, + boolean cacheTables, boolean cacheProvider) throws IOException { + for (TermProvider provider: cache) { + if (provider instanceof FieldTermProvider) { + if (provider.getRecursiveHash() == segmentReader.hashCode() + && ((FieldTermProvider)provider).getRequest().equals(request)) { + return (FieldTermProvider)provider; + } + } + } + FieldTermProvider provider = + new FieldTermProvider(segmentReader, docIDBase, request, cacheTables); + if (cacheProvider) { + cache.add(provider); + } + return provider; + } + + public void purgeAllCaches() { + if (ExposedSettings.debug) { + System.out.println("ExposedCache.purgeAllCaches() called"); + } + cache.clear(); + for (PurgeCallback purger: remoteCaches) { + purger.purgeAllCaches(); + } + } + + /** + * Purges all entries with connections to the given index reader. For exposed + * structures that means all structures except the ones that are + * FieldTermProviders not relying on the given index reader. + * @param r the reader to purge. + */ + public synchronized void purge(IndexReader r) { + if (ExposedSettings.debug) { + System.out.println("ExposedCache.purge(" + r + ") called"); + } + Iterator remover = + cache.iterator(); + while (remover.hasNext()) { + TermProvider provider = remover.next(); + if (!(provider instanceof FieldTermProvider) || + provider.getRecursiveHash() == r.hashCode()) { + remover.remove(); + } + } + for (PurgeCallback purger: remoteCaches) { + purger.purge(r); + } + } + + public static interface PurgeCallback { + void purgeAllCaches(); + void purge(IndexReader r); + } + + public String toString() { + StringWriter sw = new StringWriter(); + sw.append("ExposedCache(\n"); + for (TermProvider provider: cache) { + sw.append(" "); + sw.append(provider.toString()); + sw.append("\n"); + } + sw.append(")"); + return sw.toString(); + + } + + /** + * Release all inner cached values for the term providers for all providers + * that satisfies the given constraints. + * @param keepRoots if true, providers at the root level should release their + * inner caches. + */ + public void transitiveReleaseCaches(boolean keepRoots) { + for (TermProvider provider: cache) { + provider.transitiveReleaseCaches(0, keepRoots); + } + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedProvider.java (revision ) @@ -0,0 +1,161 @@ +package org.apache.lucene.search.exposed; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +public abstract class CachedProvider { + private final Map cache; + private int cacheSizeNum; + private int readAheadNum; + private final long finalIndex; // Do not read past this + + private boolean onlyReadAheadIfSpace = false; + private boolean stopReadAheadOnExistingValue = true; + + // Stats + private long requests = 0; + private long lookups = 0; + private long readAheadRequests = 0; + private long misses = 0; + private long lookupTime = 0; + + protected CachedProvider(int size, int readAhead, long finalIndex) { + this.cacheSizeNum = size; + this.readAheadNum = readAhead; + this.finalIndex = finalIndex; + cache = new LinkedHashMap(size, 1.2f, false) { + @Override + protected boolean removeEldestEntry( + Map.Entry eldest) { + return size() > cacheSizeNum; + } + }; + } + + protected abstract T lookup(final long index) throws IOException; + + /** + * @return short form human readable description for the wrapped provider. + */ + protected abstract String getDesignation(); + + public T get(final long index) throws IOException { + requests++; + T entry = cache.get(index); + if (entry != null) { + return entry; + } + misses++; + + long startLookup = System.nanoTime(); + entry = lookup(index); + lookupTime += System.nanoTime() - startLookup; + lookups++; + cache.put(index, entry); + readAhead(index+1, readAheadNum); + return entry; + + } + + private void readAhead(final long startIndex, final int readAhead) + throws IOException { + long index = startIndex; + int readsLeft = readAhead; + while (true) { + if (readsLeft == 0 || index > finalIndex|| + (onlyReadAheadIfSpace && cache.size() >= cacheSizeNum)) { + break; + } + readAheadRequests++; + T entry = cache.get(index); + if (entry == null) { + long startLookup = System.nanoTime(); + entry = lookup(index); + lookupTime += System.nanoTime() - startLookup; + lookups++; + if (entry == null) { + break; + } +/* if (index > 19999) { + System.out.println("Putting " + index); + }*/ + cache.put(index, entry); + } else if (stopReadAheadOnExistingValue) { + break; + } + index++; + readsLeft--; + } + } + + /** + * Removes the cache entry for the given index from cache is present. + *

+ * If the user of the cache knows that the String for a given ordinal should + * not be used again, calling this method helps the cache to perform better. + * @param index the ordinal for the elementto remove. + * @return the old element if it was present in the cache. + */ + public T release(final long index) { + return cache.remove(index); + } + + /** + * Clears the cache. + */ + public void clear() { + cache.clear(); + requests = 0; + lookups = 0; + readAheadRequests = 0; + misses = 0; + lookupTime = 0; + } + + public int getCacheSize() { + return cacheSizeNum; + } + + /** + * Setting the cache size clears the cache. + * @param cacheSize the new size of the cache, measured in elements. + */ + public void setCacheSize(int cacheSize) { + this.cacheSizeNum = cacheSize; + cache.clear(); + } + + public int getReadAhead() { + return readAheadNum; + } + + public void setReadAhead(int readAhead) { + this.readAheadNum = readAhead; + } + + public boolean isOnlyReadAheadIfSpace() { + return onlyReadAheadIfSpace; + } + + public void setOnlyReadAheadIfSpace(boolean onlyReadAheadIfSpace) { + this.onlyReadAheadIfSpace = onlyReadAheadIfSpace; + } + + public boolean isStopReadAheadOnExistingValue() { + return stopReadAheadOnExistingValue; + } + + public void setStopReadAheadOnExistingValue( + boolean stopReadAheadOnExistingValue) { + this.stopReadAheadOnExistingValue = stopReadAheadOnExistingValue; + } + + public String getStats() { + return getDesignation() + " cacheSize=" + cacheSizeNum + + ", misses=" + misses + "/" + requests + + ", lookups=" + lookups + " (" + lookupTime / 1000000 + " ms ~= " + + (lookupTime == 0 ? "N/A" : lookups * 1000000 / lookupTime) + + " lookups/ms), readAheads=" + readAheadRequests; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FixedGapCodec.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FixedGapCodec.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/FixedGapCodec.java (revision ) @@ -0,0 +1,150 @@ +package org.apache.lucene.search.exposed; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.*; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.Set; + +/** Adapted from Default codec (VariableGap). */ +public class FixedGapCodec extends Codec { + + public FixedGapCodec() { + super("FixedGap"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new StandardPostingsWriter(state); + + TermsIndexWriterBase indexWriter; + boolean success = false; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docs.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); + success = true; + return ret; + } finally { + if (!success) { + try { + docs.close(); + } finally { + indexWriter.close(); + } + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId, state.context); + success = true; + } finally { + if (!success) { + postings.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader( + indexReader, state.dir, state.fieldInfos, + state.segmentInfo.name, postings, + state.context, TERMS_CACHE_SIZE, + state.codecId); + success = true; + return ret; + } finally { + if (!success) { + try { + postings.close(); + } finally { + indexReader.close(); + } + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, id, files); + BlockTermsReader.files(dir, segmentInfo, id, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); + } + + @Override + public void getExtensions(Set extensions) { + getStandardExtensions(extensions); + } + + public static void getStandardExtensions(Set extensions) { + extensions.add(FREQ_EXTENSION); + extensions.add(PROX_EXTENSION); + BlockTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer( + // TODO: Check compound option + state, BytesRef.getUTF8SortedAsUnicodeComparator(), true); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + // TODO: Check compound option + return new DefaultDocValuesProducer( + state.segmentInfo, state.dir, state.fieldInfos, state.codecId, true, + BytesRef.getUTF8SortedAsUnicodeComparator(), state.context); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java (revision ) @@ -0,0 +1,330 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Keeps track of a group containing 1 or more providers ensuring term + * uniqueness and ordering. The providers are normally either + * {@link FieldTermProvider} or {@link GroupTermProvider}. + *

+ * The standard use case calls for caching the GroupTermProvider at segment + * level as group setups rarely changes. + */ +// TODO: Extend to handle term ordinals > Integer.MAX_VALUE +public class GroupTermProvider extends TermProviderImpl { + private final List providers; + private final ExposedRequest.Group request; + private final int readerHash; + + private PackedInts.Reader order; + + // Starting points for term ordinals in the providers list + private final long[] termOrdinalStarts; + + // FIXME: this should be relative to segments, not providers. + //Current implementation is not valid for multiple fields in the same segment + private final long[] docIDStarts; + + public GroupTermProvider(int readerHash, + List providers, ExposedRequest.Group request, + boolean cacheTables) throws IOException { + super(null, 0, request.getComparator(), request.getComparatorID(), + "Group " + request.getName(), cacheTables); + this.readerHash = readerHash; + this.providers = checkProviders(providers); + this.request = request; + + long[][] starts = calculateStarts(); + docIDStarts = starts[0]; + termOrdinalStarts = starts[1]; + } + + private List checkProviders(List providers) { + for (TermProvider provider: providers) { + try { + provider.getDocIDBase(); + } catch (UnsupportedOperationException e) { + throw new IllegalArgumentException( + "The GrouptermProvider currently only supports providers that are " + + "capable of stating their docIDBase. Received " + provider); + } + } + return providers; + } + + /** + * The starts are straight forward for term ordinals as they are just mapped + * as one long list of ordinals in the source order. We can do there is no + * authoritative index-level ordinal mapping. + *

+ * Document IDs are a bit trickier as they are used at the index-level. + * We need to handle the case where more than one field from the same segment + * make up a group. In that case the docIDs should have the same start. + *

+ * Important: In order for the calculator to work, the sources must be + * monotonic increasing + * @throws IOException if the underlying providers failed. + * @return the calculated starts. doc, term. + */ + private long[][] calculateStarts() throws IOException { + long sanityCheck = 0; + long[] termOrdinalStarts = new long[providers.size() + 1]; + long[] docIDStarts = new long[providers.size() + 1]; + + for (int i = 1 ; i <= providers.size() ; i++) { + termOrdinalStarts[i] = + termOrdinalStarts[i-1] + providers.get(i-1).getOrdinalTermCount(); + sanityCheck += providers.get(i-1).getOrdinalTermCount(); + } + if (sanityCheck > Integer.MAX_VALUE-1) { + throw new InternalError("There are " + sanityCheck + " terms in total in " + + "the underlying TermProviders which is more than the current limit " + + "of Integer.MAX_VALUE"); + } + + long docStart = 0; + for (int i = 0 ; i < providers.size() ; i++) { + TermProvider provider = providers.get(i); + docIDStarts[i] = provider.getDocIDBase(); + docStart = provider.getDocIDBase() + provider.getMaxDoc(); + } + docIDStarts[docIDStarts.length-1] = docStart; + +/* long lastHash = -1; + long docStart = 0; + for (int i = 0 ; i < providers.size() ; i++) { + docIDStarts[i] = docStart; + TermProvider provider = providers.get(i); + docStart += provider.getRecursiveHash() == lastHash ? 0 : + provider.getMaxDoc(); + lastHash = provider.getRecursiveHash(); + } + docIDStarts[docIDStarts.length-1] = docStart;*/ + long[][] result = new long[2][]; + result[0] = docIDStarts; + result[1] = termOrdinalStarts; + return result; + } + + public ExposedRequest.Group getRequest() { + return request; + } + + public String getField(long ordinal) throws IOException { + int providerIndex = getProviderIndex(ordinal); + return providers.get(providerIndex). + getField(adjustOrdinal(ordinal, providerIndex)); + } + + private int getProviderIndex(long ordinal) throws IOException { + for (int i = 1 ; i < termOrdinalStarts.length ; i++) { + if (ordinal < termOrdinalStarts[i]) { + return i-1; + } + } + throw new IllegalArgumentException("The term ordinal " + ordinal + + " is above the maximum " + (getOrdinalTermCount() - 1)); + } + + private long adjustOrdinal(long ordinal, int providerIndex) { + return providerIndex == 0 ? ordinal : + ordinal - termOrdinalStarts[providerIndex]; + } + + public synchronized BytesRef getTerm(long ordinal) throws IOException { + int providerIndex = getProviderIndex(ordinal); + return providers.get(providerIndex). + getTerm(adjustOrdinal(ordinal, providerIndex)); + } + + public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException { + int providerIndex = getProviderIndex(ordinal); + return providers.get(providerIndex). + getDocsEnum(adjustOrdinal(ordinal, providerIndex), reuse); + } + + public String getOrderedField(long indirect) throws IOException { + return getField(getOrderedOrdinals().get((int)indirect)); + } + + public BytesRef getOrderedTerm(final long indirect) throws IOException { + return indirect == -1 ? null : + getTerm(getOrderedOrdinals().get((int)indirect)); + } + + /** + * Note that this method calculates ordered ordinals to determine the unique + * terms. + * @return The number of unique terms. + * @throws IOException if the underlyind providers failed. + */ + public long getUniqueTermCount() throws IOException { + return getOrderedOrdinals().size(); + } + + public long getOrdinalTermCount() throws IOException { + return termOrdinalStarts[termOrdinalStarts.length-1]; + } + + public long getMaxDoc() { + return docIDStarts[docIDStarts.length-1]; + } + + @Override + public IndexReader getReader() { + throw new UnsupportedOperationException( + "Cannot request a reader from a collection of readers"); + } + + public int getReaderHash() { + return readerHash; + } + + public int getRecursiveHash() { + int hash = 0; + for (TermProvider provider: providers) { + hash += provider.hashCode(); + } + return hash; + } + + public PackedInts.Reader getOrderedOrdinals() throws IOException { + if (order != null) { + return order; + } + PackedInts.Reader newOrder; +/* if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID())) { + // TODO: This produces duplicates! We need to sort + newOrder = new IdentityReader((int)getOrdinalTermCount()); + } else {*/ + long sortTime = -System.currentTimeMillis(); + newOrder = sortOrdinals(); + sortTime += System.currentTimeMillis(); +/* if (ExposedSettings.debug) { + System.out.println("Merge sorted group " + getDesignation() + + " with " + ExposedUtil.time("ordinals", newOrder.size(), sortTime)); + }*/ + if (cacheTables) { + order = newOrder; + } + return newOrder; + } + + private PackedInts.Reader sortOrdinals() throws IOException { +// System.out.println("FacetGroup sorting ordinals from " + providers.size() +// + " providers"); + int maxTermCount = (int)termOrdinalStarts[termOrdinalStarts.length-1]; + long iteratorConstruction = System.currentTimeMillis(); + PackedInts.Mutable collector = ExposedSettings.getMutable( + maxTermCount, maxTermCount); + //new GrowingMutable(0, maxTermCount, 0, maxTermCount); + long iteratorTime = -System.currentTimeMillis(); + Iterator iterator = getIterator(false); + iteratorTime += System.currentTimeMillis(); +// System.out.println("Group " + getDesignation() + " iterator constructed in " +// + iteratorTime + "ms"); + + iteratorConstruction = System.currentTimeMillis() - iteratorConstruction; + if (ExposedSettings.debug) { + System.out.println("Group total iterator construction: " + + ExposedUtil.time("ordinals", maxTermCount, iteratorConstruction)); + } + + int uniqueTermCount = 0; + long extractionTime = -System.currentTimeMillis(); + while (iterator.hasNext()) { + ExposedTuple tuple = iterator.next(); +// System.out.println("sortOrdinals " + tuple + " term = " + tuple.term.utf8ToString() + " lookup term " + getTerm(tuple.ordinal).utf8ToString()); + collector.set((int)tuple.indirect, tuple.ordinal); + uniqueTermCount++; +// collector.set(uniqueTermCount++, tuple.indirect); + } +// System.out.println("Sorted merged term ordinals to " + collector); + + long reducetime = -System.currentTimeMillis(); + PackedInts.Mutable result = ExposedSettings.getMutable( + uniqueTermCount, maxTermCount); + for (int i = 0 ; i < uniqueTermCount ; i++) { + result.set(i, collector.get(i)); + } + reducetime += System.currentTimeMillis(); + + extractionTime += System.currentTimeMillis(); + if (ExposedSettings.debug) { + System.out.println("Group ordinal iterator depletion from " + + providers.size() + " providers: " + + ExposedUtil.time("ordinals", result.size(), extractionTime) + + " (Memory optimize time: " + reducetime + " ms)"); + } + return result; + } + + + @Override + public Iterator getIterator(boolean collectDocIDs) + throws IOException { + return new MergingTermDocIterator( + this, providers, request.getComparator(), collectDocIDs); + } + + public long segmentToIndexDocID(int providerIndex, long segmentDocID) { + return docIDStarts[providerIndex] + segmentDocID; + } + + public long segmentToIndexTermOrdinal( + int providerIndex, long segmentTermOrdinal) { + return termOrdinalStarts[providerIndex] + segmentTermOrdinal; + } + + @Override + public int getDocIDBase() { + throw new UnsupportedOperationException( + "No docIDBase can be inferred from GroupTermProvider"); + } + + public String toString() { + if (order == null) { + return "GroupTermProvider(" + request.getName() + ", #subProviders=" + + providers.size() + ", no order cached, " + super.toString() + ")"; + } + return "GroupTermProvider(" + request.getName() + ", #subProviders=" + + providers.size() + ", order.length=" + order.size() + " mem=" + + packedSize(order) + ", " + super.toString() + ")"; + } + + @Override + public void transitiveReleaseCaches(int level, boolean keepRoot) { + if (!keepRoot || level > 0) { + order = null; + } + level++; + super.transitiveReleaseCaches(level, keepRoot); + for (TermProvider provider: providers) { + provider.transitiveReleaseCaches(level, keepRoot); + } + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/MonotonicReaderFactory.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/MonotonicReaderFactory.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/MonotonicReaderFactory.java (revision ) @@ -0,0 +1,27 @@ +package org.apache.lucene.util.packed; + +/** + * Takes a reader with monotonic increasing or decreasing values and produces + * a reduced memory reader with the same content. The amount of memory reduction + * depends on the layout of the given Reader. With a uniform increment of + * values the savings are substantial. If increments changes between very small + * and very large increments, the MonotonicReader might use more memory. + *

+ * The cost of reducing the memory requirements is a small increase in access + * time. + */ +// TODO: Implement this +public class MonotonicReaderFactory { + + /** + * Attempts to create a new reader with the same content but with reduced + * memory requirements. + * @param reader a monotonic increasing or decreasing reader. + * @return a new Reader with the same content or the given reader if it was + * not possible to produce a new reader with smaller memory requirements. + */ + public static PackedInts.Reader reduce(PackedInts.Reader reader) { + throw new UnsupportedOperationException("Not implemented yet"); + } + +} Index: solr/contrib/exposed/src/test/resources/solr/conf/solrconfig-exposed.xml =================================================================== --- solr/contrib/exposed/src/test/resources/solr/conf/solrconfig-exposed.xml (revision ) +++ solr/contrib/exposed/src/test/resources/solr/conf/solrconfig-exposed.xml (revision ) @@ -0,0 +1,511 @@ + + + + + + + + + + + + ${solr.data.dir:./solr/data} + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + false + 10 + + + + 32 + 2147483647 + 10000 + 1000 + 10000 + + + + + + + + 1000 + 10000 + + single + + + + + false + 8 + + 10 + 2147483647 + 10000 + + + true + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + dismax + + + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + false + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + false + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + false + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + false + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + termsComp + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + false + true + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + + true + non_indexed_signature_sS + false + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + + + + 10 + 5 + 5 + + + + + + exposedComponent + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFactory.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFactory.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedFactory.java (revision ) @@ -0,0 +1,92 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +// TODO: Is this used anymore? +public class ExposedFactory { + + // TODO: Split this so that group is explicit + public static TermProvider createProvider( + IndexReader reader, String groupName, List fieldNames, + Comparator comparator, boolean reverse, String comparatorID) + throws IOException { + if (fieldNames.size() == 0) { + throw new IllegalArgumentException("There must be at least 1 field name"); + } + if (reader.getSequentialSubReaders() == null && fieldNames.size() == 1) { + // Segment-level single field + ExposedRequest.Field request = new ExposedRequest.Field( + fieldNames.get(0), comparator, reverse, comparatorID); + return new FieldTermProvider(reader, 0, request, true); + } + + if (reader.getSequentialSubReaders() == null && fieldNames.size() >= 1) { + // Segment-level multi field + List fieldProviders = + new ArrayList(fieldNames.size()); + List fieldRequests = + new ArrayList(fieldNames.size()); + for (String fieldName: fieldNames) { + ExposedRequest.Field fieldRequest = new ExposedRequest.Field( + fieldName, comparator, reverse, comparatorID); + fieldRequests.add(fieldRequest); + fieldProviders.add(new FieldTermProvider( + reader, 0, fieldRequest, false)); + } + ExposedRequest.Group groupRequest = new ExposedRequest.Group( + groupName, fieldRequests, comparator, reverse, comparatorID); + return new GroupTermProvider( + reader.hashCode(), fieldProviders, groupRequest, true); + } + if (reader.getSequentialSubReaders() != null && fieldNames.size() == 1) { + // Index-level single field + IndexReader[] subs = reader.getSequentialSubReaders(); + List fieldProviders = + new ArrayList(subs.length); + List fieldRequests = + new ArrayList(subs.length); + int docBase = 0; + for (IndexReader sub: subs) { + ExposedRequest.Field fieldRequest = new ExposedRequest.Field( + fieldNames.get(0), comparator, reverse, comparatorID); + fieldRequests.add(fieldRequest); + fieldProviders.add(new FieldTermProvider( + sub, docBase, fieldRequest, false)); + //sub.getTopReaderContext().docBaseInParent + docBase += sub.maxDoc(); + } + ExposedRequest.Group groupRequest = new ExposedRequest.Group( + groupName, fieldRequests, comparator, reverse, comparatorID); + return new GroupTermProvider( + reader.hashCode(), fieldProviders, groupRequest, true); + } + // Index-level multi field + IndexReader[] subs = reader.getSequentialSubReaders(); + List fieldProviders = + new ArrayList(subs.length * fieldNames.size()); + List fieldRequests = + new ArrayList(subs.length * fieldNames.size()); + int docBase = 0; + for (IndexReader sub: subs) { + for (String fieldName: fieldNames) { + ExposedRequest.Field fieldRequest = new ExposedRequest.Field( + fieldName, comparator, reverse, comparatorID); + fieldRequests.add(fieldRequest); + fieldProviders.add(new FieldTermProvider( + sub, docBase, fieldRequest, false)); + } + //sub.getTopReaderContext().docBaseInParent + docBase += sub.maxDoc(); + } + ExposedRequest.Group groupRequest = new ExposedRequest.Group( + groupName, fieldRequests, comparator, reverse, comparatorID); + return new GroupTermProvider( + reader.hashCode(), fieldProviders, groupRequest, true); + } +} Index: solr/contrib/exposed/README.txt =================================================================== --- solr/contrib/exposed/README.txt (revision ) +++ solr/contrib/exposed/README.txt (revision ) @@ -0,0 +1,50 @@ +Getting Started +--------------- +The Exposed faceting system can be tested with the standard Solr example the following way: + +1. Run 'ant example' in the solr folder. +1.1 mkdir example/solr/lib +1.2 cp dist/apache-solr-exposed-4.0-SNAPSHOT.jar example/solr/lib/ + + +2. Start Solr from the example folder with + java -jar -start.jar + + +3. Use the Tcl script facet_samples.tcl from contrib/exposed to create sample data: +tclsh ./facet_samples.tcl 100000 2 2 -u 100 100 > 100000.csv + + +4. Index the sample data with curl: + +curl "http://localhost:8983/solr/update/csv?commit=true&optimize=true" --data-binary @100000.csv -H 'Content-type:text/plain; charset=utf-8' + + +5. Verify that exposed faceting works without hierarchical + +http://localhost:8983/solr/select/?qt=exprh&efacet=true&efacet.field=path_ss&q=*%3A*&fl=id&version=2.2&start=0&rows=10&indent=on + + +10. Verify that exposed faceting works with hierarchical: + +http://localhost:8983/solr/select/?qt=exprh&efacet=true&efacet.field=path_ss&efacet.hierarchical=true&q=*%3A*&fl=id&version=2.2&start=0&rows=10&indent=on + +11. Play around, but note the show stopper described below. + +----------------------------------------- +TODO +----------------------------------------- + +This patch is extremely rough around the edges and is more a proof of concept than a real +Solr plugin. While the Lucene-part seems fairly solid, all the Solr-parts are hacked together. + +Proper integration with Solr is ongoing. The show stopper as of march 2011 is +* No detection of index changes: Updating the index means that everything fails. + Relevant wiki pages seems to be + http://wiki.apache.org/solr/SolrCaching and http://wiki.apache.org/solr/SolrPlugins#CacheRegenerator + +Other problems are +* No field-specific parameters, even though the JavaDoc says that they work. +* Bad build scripting as I do not understand the ant files. + +- Toke Eskildsen, te@statsbiblioteket.dk Index: lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenizer.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenizer.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/analysis/MockTokenizer.java (revision ) @@ -0,0 +1,204 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.util.AttributeSource.AttributeFactory; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; + +/** + * Tokenizer for testing. + *

+ * This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD} + * tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test + * it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior: + *

    + *
  • An internal state-machine is used for checking consumer consistency. These checks can + * be disabled with {@link #setEnableChecks(boolean)}. + *
  • For convenience, optionally lowercases terms that it outputs. + *
+ */ +public class MockTokenizer extends Tokenizer { + /** Acts Similar to WhitespaceTokenizer */ + public static final CharacterRunAutomaton WHITESPACE = + new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton()); + /** Acts Similar to KeywordTokenizer. + * TODO: Keyword returns an "empty" token for an empty reader... + */ + public static final CharacterRunAutomaton KEYWORD = + new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); + /** Acts like LetterTokenizer. */ + // the ugly regex below is Unicode 5.2 [:Letter:] + public static final CharacterRunAutomaton SIMPLE = + new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); + + private final CharacterRunAutomaton runAutomaton; + private final boolean lowerCase; + private final int maxTokenLength; + public static final int DEFAULT_MAX_TOKEN_LENGTH = Integer.MAX_VALUE; + private int state; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + int off = 0; + + // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? + // currently, we can only check that the lifecycle is correct if someone is reusing, + // but not for "one-offs". + private static enum State { + SETREADER, // consumer set a reader input either via ctor or via reset(Reader) + RESET, // consumer has called reset() + INCREMENT, // consumer is consuming, has called incrementToken() == true + INCREMENT_FALSE, // consumer has called incrementToken() which returned false + END, // consumer has called end() to perform end of stream operations + CLOSE // consumer has called close() to release any resources + }; + + private State streamState = State.CLOSE; + private boolean enableChecks = true; + + public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { + super(factory, input); + this.runAutomaton = runAutomaton; + this.lowerCase = lowerCase; + this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; + this.maxTokenLength = maxTokenLength; + } + + public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { + this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, runAutomaton, lowerCase, maxTokenLength); + } + + public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { + this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH); + } + + @Override + public final boolean incrementToken() throws IOException { + assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT) + : "incrementToken() called while in wrong state: " + streamState; + clearAttributes(); + for (;;) { + int startOffset = off; + int cp = readCodePoint(); + if (cp < 0) { + break; + } else if (isTokenChar(cp)) { + int endOffset; + do { + char chars[] = Character.toChars(normalize(cp)); + for (int i = 0; i < chars.length; i++) + termAtt.append(chars[i]); + endOffset = off; + if (termAtt.length() >= maxTokenLength) { + break; + } + cp = readCodePoint(); + } while (cp >= 0 && isTokenChar(cp)); + offsetAtt.setOffset(correctOffset(startOffset), correctOffset(endOffset)); + streamState = State.INCREMENT; + return true; + } + } + streamState = State.INCREMENT_FALSE; + return false; + } + + protected int readCodePoint() throws IOException { + int ch = input.read(); + if (ch < 0) { + return ch; + } else { + assert !Character.isLowSurrogate((char) ch); + off++; + if (Character.isHighSurrogate((char) ch)) { + int ch2 = input.read(); + if (ch2 >= 0) { + off++; + assert Character.isLowSurrogate((char) ch2); + return Character.toCodePoint((char) ch, (char) ch2); + } + } + return ch; + } + } + + protected boolean isTokenChar(int c) { + state = runAutomaton.step(state, c); + if (state < 0) { + state = runAutomaton.getInitialState(); + return false; + } else { + return true; + } + } + + protected int normalize(int c) { + return lowerCase ? Character.toLowerCase(c) : c; + } + + @Override + public void reset() throws IOException { + super.reset(); + state = runAutomaton.getInitialState(); + off = 0; + assert !enableChecks || streamState != State.RESET : "double reset()"; + streamState = State.RESET; + } + + @Override + public void close() throws IOException { + super.close(); + // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close() + // these tests should disable this check, by default we check the normal workflow. + // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this + assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState; + streamState = State.CLOSE; + } + + @Override + public void reset(Reader input) throws IOException { + super.reset(input); + assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; + streamState = State.SETREADER; + } + + @Override + public void end() throws IOException { + int finalOffset = correctOffset(off); + offsetAtt.setOffset(finalOffset, finalOffset); + // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false. + // these tests should disable this check (in general you should consume the entire stream) + assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + streamState = State.END; + } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestSubtags.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestSubtags.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestSubtags.java (revision ) @@ -0,0 +1,102 @@ +/* $Id:$ + * + * The Summa project. + * Copyright (C) 2005-2010 The State and University Library + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.apache.lucene.search.exposed.facet.request; + +import org.apache.lucene.search.exposed.facet.ParseHelper; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +class FacetRequestSubtags implements SubtagsConstraints { + final int maxTags; + final int minCount; + final int minTotalCount; + final SUBTAGS_ORDER order; + final FacetRequestSubtags subtags; + + public FacetRequestSubtags(int maxTags, int minCount, int minTotalCount, + SUBTAGS_ORDER order) { + this.maxTags = maxTags; + this.minCount = minCount; + this.minTotalCount = minTotalCount; + this.order = order; + this.subtags = null; + } + + public FacetRequestSubtags(XMLStreamReader reader, SubtagsConstraints defaults) + throws XMLStreamException { + int maxTags = defaults.getMaxTags(); + int minCount = defaults.getMinCount(); + int minTotalCount = defaults.getMinTotalCount(); + SUBTAGS_ORDER order = defaults.getSubtagsOrder(); + + final String request = "Not available"; + for (int i = 0 ; i < reader.getAttributeCount() ; i++) { + String attribute = reader.getAttributeLocalName(i); + String value = reader.getAttributeValue(i); + if ("maxtags".equals(attribute)) { + maxTags = ParseHelper.getInteger(request, "maxtags", value); + if (maxTags == -1) { + maxTags = Integer.MAX_VALUE; + } + } else if ("mincount".equals(attribute)) { + minCount = ParseHelper.getInteger(request, "mincount", value); + } else if ("mintotalcount".equals(attribute)) { + minTotalCount = ParseHelper.getInteger(request, "mintotalcount", value); + } else if ("suborder".equals(attribute)) { + order = SUBTAGS_ORDER.fromString(value); + } + } + this.maxTags = maxTags; + this.minCount = minCount; + this.minTotalCount = minTotalCount; + this.order = order; + + reader.nextTag(); + FacetRequestSubtags subtags = null; + while (!ParseHelper.atEnd(reader, "subtags")) { + if (ParseHelper.atStart(reader, "subtags")) { + subtags = new FacetRequestSubtags(reader, this); + } + reader.nextTag(); + } + this.subtags = subtags; + } + + public int getMaxTags() { + return maxTags; + } + + public int getMinCount() { + return minCount; + } + + public int getMinTotalCount() { + return minTotalCount; + } + + public SUBTAGS_ORDER getSubtagsOrder() { + return order; + } + + public SubtagsConstraints getDeeperLevel() { + return subtags == null ? this : subtags; + } +} Index: solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetParams.java =================================================================== --- solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetParams.java (revision ) +++ solr/contrib/exposed/src/java/org/apache/solr/exposed/ExposedFacetParams.java (revision ) @@ -0,0 +1,114 @@ +package org.apache.solr.exposed; + +/** + * Exposed Facet Parameters, taken from + * {@link org.apache.solr.common.params.FacetParams}. + */ +public interface ExposedFacetParams { + /** + * Should facet counts be calculated? + */ + public static final String EFACET = "efacet"; + + /** + * Any lucene formated queries the user would like to use for + * Facet Constraint Counts (multi-value) + */ +// public static final String EFACET_QUERY = EFACET + ".query"; + /** + * Any field whose terms the user wants to enumerate over for + * Facet Constraint Counts (multi-value) + */ + public static final String EFACET_FIELD = EFACET + ".field"; + + /** + * The offset into the list of facets. + * Can be overridden on a per field basis. + */ + public static final String EFACET_OFFSET = EFACET + ".offset"; + + /** + * Numeric option indicating the maximum number of facet field counts + * be included in the response for each field - in descending order of count. + * Can be overridden on a per field basis. + */ + public static final String EFACET_LIMIT = EFACET + ".limit"; + + /** + * Numeric option indicating the minimum number of hits before a facet should + * be included in the response. Can be overridden on a per field basis. + */ + public static final String EFACET_MINCOUNT = EFACET + ".mincount"; + + /** + * Comma separated list of fields to pivot + * + * example: author,type (for types by author / types within author) + */ +// public static final String EFACET_PIVOT = EFACET + ".pivot"; + + /** + * Minimum number of docs that need to match to be included in the sublist + * + * default value is 1 + */ +// public static final String EFACET_PIVOT_MINCOUNT = EFACET_PIVOT + ".mincount"; + + + /** + * String option: "count" causes facets to be sorted + * by the count, "index" results in index order, "locale" requires + * efacet.sort.locale to be set to a valid locale. + * + * This can be overridden on a per field basis.. + */ + public static final String EFACET_SORT = EFACET + ".sort"; + + public static final String EFACET_SORT_COUNT = "count"; + public static final String EFACET_SORT_INDEX = "index"; + public static final String EFACET_SORT_LOCALE = "locale"; + + /** + * If true, the order of the tags is reversed. + */ + public static final String EFACET_REVERSE = EFACET + ".reverse"; + + + /** + * Used when efacet.sort == locale. + */ + public static final String EFACET_SORT_LOCALE_VALUE = EFACET + ".sort.locale"; + + + /** + * If true, the facets are treated as hierarchical. + * + * This can be overridden on a per field basis.. + */ + public static final String EFACET_HIERARCHICAL = EFACET + ".hierarchical"; + + /** + * The delimiter when using hierarchical faceting. Default is '/'. + * + * This can be overridden on a per field basis. + */ + public static final String EFACET_HIERARCHICAL_DELIMITER = + EFACET + ".hierarchical.delimiter"; + public static final String EFACET_HIERARCHICAL_DELIMITER_DEFAULT = "/"; + + /** + * The maximum depth to expand hierarchical faceting to. 1 is equivalent to + * simple faceting. Default is unlimited. + * + * This can be overridden on a per field basis. + */ + public static final String EFACET_HIERARCHICAL_LEVELS = + EFACET + ".hierarchical.levels"; + + // TODO: Mimic params from SOLR-64 + + /** + * Only return constraints of a facet field with the given prefix. + */ +// public static final String EFACET_PREFIX = EFACET + ".prefix"; +} Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/ExposedHelper.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/ExposedHelper.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/ExposedHelper.java (revision ) @@ -0,0 +1,281 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; + +import java.io.File; +import java.io.IOException; +import com.ibm.icu.text.Collator; +import java.text.DecimalFormat; +import java.util.*; + +public class ExposedHelper { + static final char[] CHARS = // Used for random content + ("abcdefghijklmnopqrstuvwxyzæøåABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ" + + "1234567890 ").toCharArray(); + public static final File INDEX_LOCATION = + new File("tmp/testfieldtermprovider"); + + // Fields in the test index + public static final String ID = "id"; // doc #0 = "0", doc #1 = "1" etc + public static final String EVEN = "even"; // "true" or "false" + public static final String ALL = "all"; // all == "all" + public static final String EVEN_NULL = "evennull"; // odd = random content + public static final String MULTI = "facet"; // 0-5 of values A to Z + + public static final DecimalFormat ID_FORMAT = new DecimalFormat("00000000"); + + public ExposedHelper() { + //deleteIndex(); + INDEX_LOCATION.mkdirs(); + } + + public void close() { + deleteIndex(); + } + private void deleteIndex() { + if (INDEX_LOCATION.exists()) { + for (File file: INDEX_LOCATION.listFiles()) { + file.delete(); + } + INDEX_LOCATION.delete(); + } + } + + // MB + public static long getMem() { + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + System.gc(); + return (Runtime.getRuntime().totalMemory() + - Runtime.getRuntime().freeMemory()) / 1048576; + } + + public static String getTime(long ms) { + if (ms < 2999) { + return ms + " ms"; + } + long seconds = Math.round(ms / 1000.0); + long minutes = seconds / 60; + return String.format("%d:%02d minutes", minutes, seconds - (minutes * 60)); + } + + public static class Pair implements Comparable { + public long docID; + public String term; + public Collator comparator; + + public Pair(long docID, String term, Collator comparator) { + this.docID = docID; + this.term = term; + this.comparator = comparator; + } + + public int compareTo(Pair o) { + return comparator.compare(term, o.term); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof Pair)) { + return false; + } + Pair other = (Pair)obj; + return term.equals(other.term) && docID == other.docID; + } + + @Override + public String toString() { + return "Pair(" + docID + ", " + term + ")"; + } + } + + public static IndexWriter getWriter() throws IOException { + return ExposedIOFactory.getWriter(INDEX_LOCATION); + } + + public static void addDocument( + IndexWriter writer, String... values) throws IOException { + Document doc = new Document(); + for (String value: values) { + String[] tokens = value.split(":"); + if (tokens.length != 2) { + throw new IllegalArgumentException("Could not split '" + value + "'"); + } + doc.add(new Field(tokens[0], tokens[1], + Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + writer.addDocument(doc); + } + + public void createIndex( + int docCount, List fields, int fieldContentLength, + int minSegments) throws IOException { + createIndex(docCount, fields, 1, fieldContentLength, 0, 6, minSegments); + } + + public void createIndex( + int docCount, List fields, int fieldFactor, + int fieldContentLength, int minFacets, int maxFacets, int minSegments) + throws IOException { + long startTime = System.nanoTime(); + File location = INDEX_LOCATION; + Random random = new Random(87); + int every = Math.max(1, docCount / 100); + + IndexWriter writer = getWriter(); + for (int docID = 0 ; docID < docCount ; docID++) { + Document doc = new Document(); + for (String field: fields) { + for (int f = 0 ; f < fieldFactor ; f++) { + doc.add(new Field( + field, + getRandomString( + random, CHARS, 1, fieldContentLength) + docID + field, + Field.Store.YES, Field.Index.NOT_ANALYZED)); + } + } + doc.add(new Field(ID, ID_FORMAT.format(docID), + Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(EVEN, docID % 2 == 0 ? "true" : "false", + Field.Store.YES, Field.Index.NOT_ANALYZED)); + int facets = random.nextInt(maxFacets-minFacets+1) + minFacets; + for (int i = 0 ; i < facets ; i++) { + doc.add(new Field(MULTI, + Character.toString((char)(random.nextInt(25) + 'A')), + Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + if (docID % 2 == 1) { + doc.add(new Field(EVEN_NULL, getRandomString( + random, CHARS, 1, fieldContentLength) + docID, + Field.Store.YES, Field.Index.NOT_ANALYZED)); + } + doc.add(new Field(ALL, ALL, Field.Store.YES, Field.Index.NOT_ANALYZED)); + writer.addDocument(doc); + if (docID == docCount / minSegments) { + writer.commit(); // Ensure minSegments + } + if (docID % every == 0) { + System.out.print("."); + } + } + System.out.print("Closing"); +// writer.optimize(); + writer.close(); + System.out.println(""); + System.out.println(String.format( + "Created %d document index with %d fields with average " + + "term length %d and total size %s in %sms at %s", + docCount, fields.size() + 2, fieldContentLength / 2, + readableSize(calculateSize(location)), + (System.nanoTime() - startTime) / 1000000, location.getAbsolutePath())); + } + + /* + Constructs a simple index with fields a, b and c. Meant for testing groups + with multiple fields. + */ + public File buildMultiFieldIndex(int docs) throws IOException { + IndexWriter w = getWriter(); + for (int docID = 0 ; docID < docs ; docID++) { + addDocument(w, + ID + ":1", + ALL + ":" + ALL, + "a:a" + docID, + "b:b" + docID / 2, + "c:c" + docID % 2); + } + w.close(true); + return INDEX_LOCATION; + } + + /* + Re-creates the test case from https://issues.apache.org/jira/browse/SOLR-475?focusedCommentId=12650071&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_12650071 + */ + public void createFacetIndex(final int docCount) throws IOException { + long startTime = System.nanoTime(); + File location = INDEX_LOCATION; + Random random = new Random(87); + + IndexWriter writer = getWriter(); + writer.getConfig().setRAMBufferSizeMB(32.0); + final int feedback = docCount / 100; + for (int docID = 0 ; docID < docCount ; docID++) { + Document doc = new Document(); + addTagsToDocument(doc, random, 10, 100); + addTagsToDocument(doc, random, 100, 10); + addTagsToDocument(doc, random, 1000, 5); + addTagsToDocument(doc, random, 10000, 5); + addTagsToDocument(doc, random, 100000, 5); + addTagsToDocument(doc, random, 100000, 10); + addTagsToDocument(doc, random, 1000000, 5); // Extra + addTagsToDocument(doc, random, 10000000, 1); // Extra + doc.add(new Field(ALL, ALL, Field.Store.YES, Field.Index.NOT_ANALYZED)); + for (int hits: new int[]{1000, 10000, 100000, 1000000, 10000000}) { + if (docCount > hits && docID % (docCount / hits) == 0) { + doc.add(new Field("hits" + hits, "true", + Field.Store.NO, Field.Index.NOT_ANALYZED)); + } + } + writer.addDocument(doc); + if (docID % feedback == 0) { + System.out.print("."); + } + } +// writer.optimize(); + writer.close(); + System.out.println(String.format( + "\nCreated %d document index with total size %s in %sms at %s", + docCount, readableSize(calculateSize(location)), + (System.nanoTime() - startTime) / 1000000, location.getAbsolutePath())); + } + + private void addTagsToDocument( + Document doc, Random random, int unique, int occurrences) { + final int num = random.nextInt(occurrences+1); + for (int i = 0 ; i < num ; i++) { + Field field = new Field("f" + unique + "_" + occurrences + "_t", + "t" + random.nextInt(unique+1), + Field.Store.NO, Field.Index.NOT_ANALYZED); + doc.add(field); + } + } + + private static StringBuffer buffer = new StringBuffer(100); + static synchronized String getRandomString( + Random random, char[] chars, int minLength, int maxLength) { + int length = minLength == maxLength ? minLength : + random.nextInt(maxLength-minLength+1) + minLength; + buffer.setLength(0); + for (int i = 0 ; i < length ; i++) { + buffer.append(chars[random.nextInt(chars.length)]); + } + return buffer.toString(); + } + + private String readableSize(long size) { + return size > 2 * 1048576 ? + size / 1048576 + "MB" : + size > 2 * 1024 ? + size / 1024 + "KB" : + size + "bytes"; + } + + private long calculateSize(File file) { + long size = 0; + if (file.isDirectory()) { + for (File sub: file.listFiles()) { + size += calculateSize(sub); + } + } else { + size += file.length(); + } + return size; + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml (revision ) @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TODO.txt =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TODO.txt (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/TODO.txt (revision ) @@ -0,0 +1,6 @@ +BUG: Reopen must reset docIDBase for FieldTermProviders + +FEATURE: Presentation tags coupled with sorting tags (1:1 mapping for any given document) +FEATURE: Optimize faceting for all documents (initial navigation point) and for no documents (index lookup) +FEATURE: Speedup by adding bulk reads to PackedInts + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestGroup.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestGroup.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequestGroup.java (revision ) @@ -0,0 +1,341 @@ +package org.apache.lucene.search.exposed.facet.request; + +import org.apache.lucene.search.exposed.ExposedComparators; +import org.apache.lucene.search.exposed.ExposedRequest; +import org.apache.lucene.search.exposed.facet.ParseHelper; +import org.apache.lucene.util.BytesRef; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +public final class FacetRequestGroup implements SubtagsConstraints { + public static final String DEFAULT_DELIMITER = "/"; + public static final int DEFAULT_LEVELS = 5; + + private final ExposedRequest.Group group; + private final FacetRequest.GROUP_ORDER order; + private final boolean reverse; + private final String locale; + private final int offset; + private final int maxTags; + private final int minCount; + private final String prefix; + private final String buildKey; + private final boolean hierarchical; + private final String delimiter; + private final int levels; + private final String startPath; + private FacetRequestSubtags subtags = null; + + // Reader must be positioned at group start + FacetRequestGroup( + XMLStreamReader reader, String request, FacetRequest.GROUP_ORDER order, + boolean defaultReverse, String locale, int maxTags, int minCount, + int offset, String prefix, boolean defaultHierarchical, int defaultLevels, + String defaultDelimiter, String defaultStartPath) + throws XMLStreamException { + String name = null; + boolean hierarchical = defaultHierarchical; + String delimiter = defaultDelimiter; + int levels = defaultLevels; + String startPath = defaultStartPath; + boolean reverse = defaultReverse; + + for (int i = 0 ; i < reader.getAttributeCount() ; i++) { + String attribute = reader.getAttributeLocalName(i); + String value = reader.getAttributeValue(i); + if ("order".equals(attribute)) { + order = FacetRequest.GROUP_ORDER.fromString(value); + } else if ("reverse".equals(attribute)) { + reverse = Boolean.parseBoolean(value); + } else if ("locale".equals(attribute)) { + locale = value; + } else if ("maxtags".equals(attribute)) { + maxTags = ParseHelper.getInteger(request, "maxtags", value); + if (maxTags == -1) { + maxTags = Integer.MAX_VALUE; + } + } else if ("mincount".equals(attribute)) { + minCount = ParseHelper.getInteger(request, "mincount", value); + } else if ("offset".equals(attribute)) { + offset = ParseHelper.getInteger(request, "offset", value); + } else if ("prefix".equals(attribute)) { + prefix = value; + } else if ("name".equals(attribute)) { + name = value; + } else if ("hierarchical".equals(attribute)) { + hierarchical = Boolean.parseBoolean(value); + } else if ("delimiter".equals(attribute)) { + delimiter = value; + } else if ("levels".equals(attribute)) { + levels = ParseHelper.getInteger(request, "levels", value); + } else if ("startpath".equals(attribute)) { + startPath = value; + } + } + this.order = order; + this.reverse = reverse; + this.locale = locale; + this.offset = offset; + this.maxTags = maxTags; + this.minCount = minCount; + this.prefix = prefix; + + this.hierarchical = hierarchical; + this.delimiter = delimiter; + this.levels = levels; + this.startPath = startPath; + + if (name == null) { + throw new XMLStreamException( + "FacetGroup name must be specified with the attribute 'name' in " + + request); + } + List fieldNames = new ArrayList(); + reader.nextTag(); + while (!ParseHelper.atEnd(reader, "group")) { + if (ParseHelper.atStart(reader, "fields")) { + reader.nextTag(); + while (!ParseHelper.atEnd(reader, "fields")) { + if (ParseHelper.atStart(reader, "field")) { + String fieldName = null; + for (int i = 0 ; i < reader.getAttributeCount() ; i++) { + if ("name".equals(reader.getAttributeLocalName(i))) { + fieldName = reader.getAttributeValue(i); + } + } + if (fieldName == null) { + throw new XMLStreamException( + "Unable to determine name for field in group " + name + + " in " + request); + } + fieldNames.add(fieldName); + } + reader.nextTag(); // Until /fields + } + } else if (ParseHelper.atStart(reader, "subtags")) { + subtags = new FacetRequestSubtags(reader, this); + } + reader.nextTag(); // until /group + } + String comparatorID = getComparatorID(order, locale); + group = createGroup( + name, fieldNames, ExposedComparators.localeToBytesRef( + locale == null ? null : new Locale(locale)), + reverse, comparatorID); + buildKey = createBuildKey(); + } + + private String getComparatorID( + FacetRequest.GROUP_ORDER order, String locale) { + String comparatorID; + if (order == FacetRequest.GROUP_ORDER.count) { + comparatorID = ExposedRequest.FREE_ORDER; + } else if (order == FacetRequest.GROUP_ORDER.index) { + comparatorID = ExposedRequest.LUCENE_ORDER; + } else if (order == FacetRequest.GROUP_ORDER.locale) { + comparatorID = new Locale(locale).toString(); + } else { + throw new IllegalArgumentException( + "The order '" + order + "' is unknown"); + } + return comparatorID; + } + + // Simple non-hierarchical constructor + public FacetRequestGroup( + ExposedRequest.Group group, FacetRequest.GROUP_ORDER order, + boolean reverse, String locale, int offset, int maxTags, int minCount, + String prefix) { + this.group = group; + this.order = order; + this.reverse = reverse; + this.locale = locale; + this.offset = offset; + this.maxTags = maxTags; + this.minCount = minCount; + this.prefix = prefix; + hierarchical = false; + delimiter = null; + levels = 1; + startPath = null; + + buildKey = createBuildKey(); + } + + // Creates a group based on a single field + public FacetRequestGroup( + String field, FacetRequest.GROUP_ORDER order, boolean reverse, + String locale, int maxTags, int minCount, int offset, String prefix, + boolean hierarchical, int levels, String delimiter, String startPath) { + Comparator comparator = ExposedComparators.localeToBytesRef( + locale == null ? null : new Locale(locale)); + String comparatorID = getComparatorID(order, locale); + List fields = new ArrayList(1); + fields.add(new ExposedRequest.Field( + field, comparator, reverse, comparatorID)); + group = new ExposedRequest.Group( + field, fields, comparator, reverse, comparatorID); + this.order = order; + this.reverse = reverse; + this.locale = locale; + this.offset = offset; + this.maxTags = maxTags; + this.minCount = minCount; + this.prefix = prefix; + this.hierarchical = hierarchical; + this.delimiter = delimiter; + this.levels = levels; + this.startPath = startPath; + + buildKey = createBuildKey(); + } + + private String createBuildKey() { + StringWriter sw = new StringWriter(); + sw.append("group(name=").append(group.getName()).append(", order="); + sw.append(order.toString()).append(", locale=").append(locale); + sw.append(", fields("); + boolean first = true; + for (String field: group.getFieldNames()) { + if (first) { + first = false; + } else { + sw.append(", "); + } + sw.append(field); + } + sw.append("), hierarchical=").append(Boolean.toString(hierarchical)); + sw.append(", delimiter=").append(delimiter).append(")"); + return sw.toString(); + } + + public String getBuildKey() { + return buildKey; + } + + public ExposedRequest.Group getGroup() { + return group; + } + + public void toXML(XMLStreamWriter out) throws XMLStreamException { + out.writeCharacters(" "); + out.writeStartElement("group"); + out.writeAttribute("name", group.getName()); + out.writeAttribute("order", order.toString()); + out.writeAttribute("reverse", Boolean.toString(reverse)); + if (locale != null) { + out.writeAttribute("locale", locale); + } + out.writeAttribute("maxtags", Integer.toString( + maxTags == Integer.MAX_VALUE ? -1 : maxTags)); + out.writeAttribute("mincount", Integer.toString(minCount)); + out.writeAttribute("offset", Integer.toString(offset)); + out.writeAttribute("prefix", prefix); + out.writeAttribute("hierarchical", Boolean.toString(hierarchical)); + out.writeAttribute("levels", Integer.toString(levels)); + out.writeAttribute("delimiter", delimiter); + out.writeAttribute("startpath", startPath == null ? "" : startPath); + out.writeCharacters("\n "); + + out.writeStartElement("fields"); + out.writeCharacters("\n"); + for (ExposedRequest.Field field: group.getFields()) { + out.writeCharacters(" "); + out.writeStartElement("field"); + out.writeAttribute("name", field.getField()); + out.writeEndElement(); + out.writeCharacters("\n"); + } + out.writeCharacters(" "); + out.writeEndElement(); // fields +// TODO: Write XML for subtags + out.writeCharacters("\n "); + out.writeEndElement(); // group + out.writeCharacters("\n"); + } + + public FacetRequest.GROUP_ORDER getOrder() { + return order; + } + + public boolean isReverse() { + return reverse; + } + + public String getLocale() { + return locale; + } + + public int getOffset() { + return offset; + } + + @Override + public int getMaxTags() { + return maxTags; + } + + @Override + public int getMinCount() { + return minCount; + } + + public String getPrefix() { + return prefix; + } + + @Override + public int getMinTotalCount() { + return minCount; + } + + @Override + public SUBTAGS_ORDER getSubtagsOrder() { + return order == + FacetRequest.GROUP_ORDER.count ? SUBTAGS_ORDER.count : SUBTAGS_ORDER.base; + } + + @Override + public SubtagsConstraints getDeeperLevel() { + return subtags == null ? this : subtags; + } + + public boolean isHierarchical() { + return hierarchical; + } + + public String getDelimiter() { + return delimiter; + } + + public int getLevels() { + return levels; + } + + public String getStartPath() { + return startPath; + } + + public static ExposedRequest.Group createGroup( + String name, List fieldNames, + Comparator comparator, boolean reverse, String comparatorID) { + if (fieldNames.size() == 0) { + throw new IllegalArgumentException("There must be at least 1 field name"); + } + List fieldRequests = + new ArrayList(fieldNames.size()); + for (String fieldName: fieldNames) { + fieldRequests.add(new ExposedRequest.Field( + fieldName, comparator, reverse, comparatorID)); + } + return new ExposedRequest.Group( + name, fieldRequests, comparator, reverse, comparatorID); + } +} Index: solr/example/solr/conf/schema.xml =================================================================== --- solr/example/solr/conf/schema.xml (revision 1141489) +++ solr/example/solr/conf/schema.xml (revision ) @@ -577,6 +577,8 @@ unknown fields indexed and/or stored by default --> + + + + + + Apache Lucene API + + + +

Apache Lucene is a high-performance, full-featured text search engine library. +Here's a simple example how to use Lucene for indexing and searching (using JUnit +to check if the results are what we expect):

+ + + + + + +
+ + + + + + + +
+ +    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
+
+    // Store the index in memory:
+    Directory directory = new RAMDirectory();
+    // To store an index on disk, use this instead:
+    //Directory directory = FSDirectory.open("/tmp/testindex");
+    IndexWriter iwriter = new IndexWriter(directory, analyzer, true,
+                                          new IndexWriter.MaxFieldLength(25000));
+    Document doc = new Document();
+    String text = "This is the text to be indexed.";
+    doc.add(new Field("fieldname", text, Field.Store.YES,
+        Field.Index.ANALYZED));
+    iwriter.addDocument(doc);
+    iwriter.close();
+    
+    // Now search the index:
+    IndexSearcher isearcher = new IndexSearcher(directory, true); // read-only=true
+    // Parse a simple query that searches for "text":
+    QueryParser parser = new QueryParser("fieldname", analyzer);
+    Query query = parser.parse("text");
+    ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
+    assertEquals(1, hits.length);
+    // Iterate through the results:
+    for (int i = 0; i < hits.length; i++) {
+      Document hitDoc = isearcher.doc(hits[i].doc);
+      assertEquals("This is the text to be indexed.", hitDoc.get("fieldname"));
+    }
+    isearcher.close();
+    directory.close(); + +
+
+ + + + + +

The Lucene API is divided into several packages:

+ + +To use Lucene, an application should: +
    +
  1. +Create Documents by +adding +Fields;
    2. + +
  2. +Create an IndexWriter +and add documents to it with addDocument();
    3. + +
  3. +Call QueryParser.parse() +to build a query from a string; and
    4. + +
  4. +Create an IndexSearcher +and pass the query to its search() +method.
    5. +
+Some simple examples of code which does this are: + +To demonstrate these, try something like: +
> java -cp lucene.jar:lucene-demo.jar org.apache.lucene.demo.IndexFiles rec.food.recipes/soups +
adding rec.food.recipes/soups/abalone-chowder +
  [ ... ] + +

> java -cp lucene.jar:lucene-demo.jar org.apache.lucene.demo.SearchFiles +
Query: chowder +
Searching for: chowder +
34 total matching documents +
1. rec.food.recipes/soups/spam-chowder +
  [ ... thirty-four documents contain the word "chowder" ... ] + +

Query: "clam chowder" AND Manhattan +
Searching for: +"clam chowder" +manhattan +
2 total matching documents +
1. rec.food.recipes/soups/clam-chowder +
  [ ... two documents contain the phrase "clam chowder" +and the word "manhattan" ... ] +
    [ Note: "+" and "-" are canonical, but "AND", "OR" +and "NOT" may be used. ]

+ +The IndexHTML demo is more sophisticated.  +It incrementally maintains an index of HTML files, adding new files as +they appear, deleting old files as they disappear and re-indexing files +as they change. +
> java -cp lucene.jar:lucene-demo.jar org.apache.lucene.demo.IndexHTML -create java/jdk1.1.6/docs/relnotes +
adding java/jdk1.1.6/docs/relnotes/SMICopyright.html +
  [ ... create an index containing all the relnotes ] +

> rm java/jdk1.1.6/docs/relnotes/smicopyright.html +

> java -cp lucene.jar:lucene-demo.jar org.apache.lucene.demo.IndexHTML java/jdk1.1.6/docs/relnotes +
deleting java/jdk1.1.6/docs/relnotes/SMICopyright.html

+ + + Index: solr/contrib/exposed/src/test/resources/solr/conf/stopwords.txt =================================================================== --- solr/contrib/exposed/src/test/resources/solr/conf/stopwords.txt (revision ) +++ solr/contrib/exposed/src/test/resources/solr/conf/stopwords.txt (revision ) @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java (revision ) @@ -0,0 +1,40 @@ +package org.apache.lucene.search.exposed; + +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RawCollationKey; + +import java.io.IOException; + +public class CachedCollatorKeyProvider extends CachedProvider { + private final TermProvider source; + private final Collator collator; + + /** + * @param source the backing term provider. + * @param collator the Collator used for key generation. + * @param cacheSize the maximum number of elements to hold in cache. + * @param readAhead the maximum number of lookups that can be performed + * after a plain lookup. + * @throws java.io.IOException if the cache could access the source. + */ + public CachedCollatorKeyProvider( + TermProvider source, Collator collator, + int cacheSize, int readAhead) throws IOException { + super(cacheSize, readAhead, source.getOrdinalTermCount()-1); + this.source = source; + this.collator = collator; + } + + @Override + protected RawCollationKey lookup(final long index) throws IOException { + RawCollationKey key = new RawCollationKey(); + return collator.getRawCollationKey( + source.getTerm(index).utf8ToString(), key); + } + + @Override + protected String getDesignation() { + return "CachedCollatorKeyProvider(" + source.getClass().getSimpleName() + + "(" + source.getDesignation() + "))"; + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java (revision ) @@ -0,0 +1,920 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.exposed; + +import java.util.Arrays; + +/** + * A stable, adaptive, iterative mergesort that requires far fewer than + * n lg(n) comparisons when running on partially sorted arrays, while + * offering performance comparable to a traditional mergesort when run + * on random arrays. Like all proper mergesorts, this sort is stable and + * runs O(n log n) time (worst case). In the worst case, this sort requires + * temporary storage space for n/2 object references; in the best case, + * it requires only a small constant amount of space. + * + * This implementation was adapted from Tim Peters's list sort for + * Python, which is described in detail here: + * + * http://svn.python.org/projects/python/trunk/Objects/listsort.txt + * + * Tim's C code may be found here: + * + * http://svn.python.org/projects/python/trunk/Objects/listobject.c + * + * The underlying techniques are described in this paper (and may have + * even earlier origins): + * + * "Optimistic Sorting and Information Theoretic Complexity" + * Peter McIlroy + * SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms), + * pp 467-474, Austin, Texas, 25-27 January 1993. + * + * While the API to this class consists solely of static methods, it is + * (privately) instantiable; a TimSort instance holds the state of an ongoing + * sort, assuming the input array is large enough to warrant the full-blown + * TimSort. Small arrays are sorted in place, using a binary insertion sort. + * + * @author Josh Bloch + * + * Modified 2010-03-22 to handle int only (specialization to save memory) + * by Toke Eskildsen (te@statsbiblioteket.dk). + */ +public class ExposedTimSort { + /** + * This is the minimum sized sequence that will be merged. Shorter + * sequences will be lengthened by calling binarySort. If the entire + * array is less than this length, no merges will be performed. + * + * This constant should be a power of two. It was 64 in Tim Peter's C + * implementation, but 32 was empirically determined to work better in + * this implementation. In the unlikely event that you set this constant + * to be a number that's not a power of two, you'll need to change the + * {@link #minRunLength} computation. + * + * If you decrease this constant, you must change the stackLen + * computation in the TimSort constructor, or you risk an + * ArrayOutOfBounds exception. See listsort.txt for a discussion + * of the minimum stack length required as a function of the length + * of the array being sorted and the minimum merge sequence length. + */ + private static final int MIN_MERGE = 32; + + /** + * The array being sorted. + */ + private final int[] a; + + /** + * The comparator for this sort. + */ + private final ExposedComparators.OrdinalComparator c; + + /** + * When we get into galloping mode, we stay there until both runs win less + * often than MIN_GALLOP consecutive times. + */ + private static final int MIN_GALLOP = 7; + + /** + * This controls when we get *into* galloping mode. It is initialized + * to MIN_GALLOP. The mergeLo and mergeHi methods nudge it higher for + * random data, and lower for highly structured data. + */ + private int minGallop = MIN_GALLOP; + + /** + * Maximum initial size of tmp array, which is used for merging. The array + * can grow to accommodate demand. + * + * Unlike Tim's original C version, we do not allocate this much storage + * when sorting smaller arrays. This change was required for performance. + */ + private static final int INITIAL_TMP_STORAGE_LENGTH = 256; + + /** + * Temp storage for merges. + */ + private int[] tmp; // Actual runtime type will be Object[], regardless of T + + /** + * A stack of pending runs yet to be merged. Run i starts at + * address base[i] and extends for len[i] elements. It's always + * true (so long as the indices are in bounds) that: + * + * runBase[i] + runLen[i] == runBase[i + 1] + * + * so we could cut the storage for this, but it's a minor amount, + * and keeping all the info explicit simplifies the code. + */ + private int stackSize = 0; // Number of pending runs on stack + private final int[] runBase; + private final int[] runLen; + + /** + * Creates a TimSort instance to maintain the state of an ongoing sort. + * + * @param a the array to be sorted + * @param c the comparator to determine the order of the sort + */ + private ExposedTimSort(int[] a, ExposedComparators.OrdinalComparator c) { + this.a = a; + this.c = c; + + // Allocate temp storage (which may be increased later if necessary) + int len = a.length; + tmp = new int[len < 2 * INITIAL_TMP_STORAGE_LENGTH ? + len >>> 1 : INITIAL_TMP_STORAGE_LENGTH]; + + /* + * Allocate runs-to-be-merged stack (which cannot be expanded). The + * stack length requirements are described in listsort.txt. The C + * version always uses the same stack length (85), but this was + * measured to be too expensive when sorting "mid-sized" arrays (e.g., + * 100 elements) in Java. Therefore, we use smaller (but sufficiently + * large) stack lengths for smaller arrays. The "magic numbers" in the + * computation below must be changed if MIN_MERGE is decreased. See + * the MIN_MERGE declaration above for more information. + */ + int stackLen = (len < 120 ? 5 : + len < 1542 ? 10 : + len < 119151 ? 19 : 40); + runBase = new int[stackLen]; + runLen = new int[stackLen]; + } + + /* + * The next two methods (which are package private and static) constitute + * the entire API of this class. Each of these methods obeys the contract + * of the public method with the same signature in java.util.Arrays. + */ + + public static void sort(int[] a, ExposedComparators.OrdinalComparator c) { + sort(a, 0, a.length, c); + } + + public static void sort(int[] a, int lo, int hi, ExposedComparators.OrdinalComparator c) { + if (c == null) { + Arrays.sort(a, lo, hi); + return; + } + + rangeCheck(a.length, lo, hi); + int nRemaining = hi - lo; + if (nRemaining < 2) + return; // Arrays of size 0 and 1 are always sorted + + // If array is small, do a "mini-TimSort" with no merges + if (nRemaining < MIN_MERGE) { + int initRunLen = countRunAndMakeAscending(a, lo, hi, c); + binarySort(a, lo, hi, lo + initRunLen, c); + return; + } + + /** + * March over the array once, left to right, finding natural runs, + * extending short natural runs to minRun elements, and merging runs + * to maintain stack invariant. + */ + ExposedTimSort ts = new ExposedTimSort(a, c); + int minRun = minRunLength(nRemaining); + do { + // Identify next run + int runLen = countRunAndMakeAscending(a, lo, hi, c); + + // If run is short, extend to min(minRun, nRemaining) + if (runLen < minRun) { + int force = nRemaining <= minRun ? nRemaining : minRun; + binarySort(a, lo, lo + force, lo + runLen, c); + runLen = force; + } + + // Push run onto pending-run stack, and maybe merge + ts.pushRun(lo, runLen); + ts.mergeCollapse(); + + // Advance to find next run + lo += runLen; + nRemaining -= runLen; + } while (nRemaining != 0); + + // Merge all remaining runs to complete sort + assert lo == hi; + ts.mergeForceCollapse(); + assert ts.stackSize == 1; + } + + /** + * Sorts the specified portion of the specified array using a binary + * insertion sort. This is the best method for sorting small numbers + * of elements. It requires O(n log n) compares, but O(n^2) data + * movement (worst case). + * + * If the initial part of the specified range is already sorted, + * this method can take advantage of it: the method assumes that the + * elements from index {@code lo}, inclusive, to {@code start}, + * exclusive are already sorted. + * + * @param a the array in which a range is to be sorted + * @param lo the index of the first element in the range to be sorted + * @param hi the index after the last element in the range to be sorted + * @param start the index of the first element in the range that is + * not already known to be sorted (@code lo <= start <= hi} + * @param c comparator to used for the sort + */ + @SuppressWarnings("fallthrough") + private static void binarySort(int[] a, int lo, int hi, int start, + ExposedComparators.OrdinalComparator c) { + assert lo <= start && start <= hi; + if (start == lo) + start++; + for ( ; start < hi; start++) { + int pivot = a[start]; + + // Set left (and right) to the index where a[start] (pivot) belongs + int left = lo; + int right = start; + assert left <= right; + /* + * Invariants: + * pivot >= all in [lo, left). + * pivot < all in [right, start). + */ + while (left < right) { + int mid = (left + right) >>> 1; + if (c.compare(pivot, a[mid]) < 0) + right = mid; + else + left = mid + 1; + } + assert left == right; + + /* + * The invariants still hold: pivot >= all in [lo, left) and + * pivot < all in [left, start), so pivot belongs at left. Note + * that if there are elements equal to pivot, left points to the + * first slot after them -- that's why this sort is stable. + * Slide elements over to make room to make room for pivot. + */ + int n = start - left; // The number of elements to move + // Switch is just an optimization for arraycopy in default case + switch(n) { + case 2: a[left + 2] = a[left + 1]; + case 1: a[left + 1] = a[left]; + break; + default: System.arraycopy(a, left, a, left + 1, n); + } + a[left] = pivot; + } + } + + /** + * Returns the length of the run beginning at the specified position in + * the specified array and reverses the run if it is descending (ensuring + * that the run will always be ascending when the method returns). + * + * A run is the longest ascending sequence with: + * + * a[lo] <= a[lo + 1] <= a[lo + 2] <= ... + * + * or the longest descending sequence with: + * + * a[lo] > a[lo + 1] > a[lo + 2] > ... + * + * For its intended use in a stable mergesort, the strictness of the + * definition of "descending" is needed so that the call can safely + * reverse a descending sequence without violating stability. + * + * @param a the array in which a run is to be counted and possibly reversed + * @param lo index of the first element in the run + * @param hi index after the last element that may be contained in the run. + It is required that @code{lo < hi}. + * @param c the comparator to used for the sort + * @return the length of the run beginning at the specified position in + * the specified array + */ + private static int countRunAndMakeAscending(int[] a, int lo, int hi, + ExposedComparators.OrdinalComparator c) { + assert lo < hi; + int runHi = lo + 1; + if (runHi == hi) + return 1; + + // Find end of run, and reverse range if descending + if (c.compare(a[runHi++], a[lo]) < 0) { // Descending + while(runHi < hi && c.compare(a[runHi], a[runHi - 1]) < 0) + runHi++; + reverseRange(a, lo, runHi); + } else { // Ascending + while (runHi < hi && c.compare(a[runHi], a[runHi - 1]) >= 0) + runHi++; + } + + return runHi - lo; + } + + /** + * Reverse the specified range of the specified array. + * + * @param a the array in which a range is to be reversed + * @param lo the index of the first element in the range to be reversed + * @param hi the index after the last element in the range to be reversed + */ + private static void reverseRange(int[] a, int lo, int hi) { + hi--; + while (lo < hi) { + int t = a[lo]; + a[lo++] = a[hi]; + a[hi--] = t; + } + } + + /** + * Returns the minimum acceptable run length for an array of the specified + * length. Natural runs shorter than this will be extended with + * {@link #binarySort}. + * + * Roughly speaking, the computation is: + * + * If n < MIN_MERGE, return n (it's too small to bother with fancy stuff). + * Else if n is an exact power of 2, return MIN_MERGE/2. + * Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k + * is close to, but strictly less than, an exact power of 2. + * + * For the rationale, see listsort.txt. + * + * @param n the length of the array to be sorted + * @return the length of the minimum run to be merged + */ + private static int minRunLength(int n) { + assert n >= 0; + int r = 0; // Becomes 1 if any 1 bits are shifted off + while (n >= MIN_MERGE) { + r |= (n & 1); + n >>= 1; + } + return n + r; + } + + /** + * Pushes the specified run onto the pending-run stack. + * + * @param runBase index of the first element in the run + * @param runLen the number of elements in the run + */ + private void pushRun(int runBase, int runLen) { + this.runBase[stackSize] = runBase; + this.runLen[stackSize] = runLen; + stackSize++; + } + + /** + * Examines the stack of runs waiting to be merged and merges adjacent runs + * until the stack invariants are reestablished: + * + * 1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1] + * 2. runLen[i - 2] > runLen[i - 1] + * + * This method is called each time a new run is pushed onto the stack, + * so the invariants are guaranteed to hold for i < stackSize upon + * entry to the method. + */ + private void mergeCollapse() { + while (stackSize > 1) { + int n = stackSize - 2; + if (n > 0 && runLen[n-1] <= runLen[n] + runLen[n+1]) { + if (runLen[n - 1] < runLen[n + 1]) + n--; + mergeAt(n); + } else if (runLen[n] <= runLen[n + 1]) { + mergeAt(n); + } else { + break; // Invariant is established + } + } + } + + /** + * Merges all runs on the stack until only one remains. This method is + * called once, to complete the sort. + */ + private void mergeForceCollapse() { + while (stackSize > 1) { + int n = stackSize - 2; + if (n > 0 && runLen[n - 1] < runLen[n + 1]) + n--; + mergeAt(n); + } + } + + /** + * Merges the two runs at stack indices i and i+1. Run i must be + * the penultimate or antepenultimate run on the stack. In other words, + * i must be equal to stackSize-2 or stackSize-3. + * + * @param i stack index of the first of the two runs to merge + */ + private void mergeAt(int i) { + assert stackSize >= 2; + assert i >= 0; + assert i == stackSize - 2 || i == stackSize - 3; + + int base1 = runBase[i]; + int len1 = runLen[i]; + int base2 = runBase[i + 1]; + int len2 = runLen[i + 1]; + assert len1 > 0 && len2 > 0; + assert base1 + len1 == base2; + + /* + * Record the length of the combined runs; if i is the 3rd-last + * run now, also slide over the last run (which isn't involved + * in this merge). The current run (i+1) goes away in any case. + */ + runLen[i] = len1 + len2; + if (i == stackSize - 3) { + runBase[i + 1] = runBase[i + 2]; + runLen[i + 1] = runLen[i + 2]; + } + stackSize--; + + /* + * Find where the first element of run2 goes in run1. Prior elements + * in run1 can be ignored (because they're already in place). + */ + int k = gallopRight(a[base2], a, base1, len1, 0, c); + assert k >= 0; + base1 += k; + len1 -= k; + if (len1 == 0) + return; + + /* + * Find where the last element of run1 goes in run2. Subsequent elements + * in run2 can be ignored (because they're already in place). + */ + len2 = gallopLeft(a[base1 + len1 - 1], a, base2, len2, len2 - 1, c); + assert len2 >= 0; + if (len2 == 0) + return; + + // Merge remaining runs, using tmp array with min(len1, len2) elements + if (len1 <= len2) + mergeLo(base1, len1, base2, len2); + else + mergeHi(base1, len1, base2, len2); + } + + /** + * Locates the position at which to insert the specified key into the + * specified sorted range; if the range contains an element equal to key, + * returns the index of the leftmost equal element. + * + * @param key the key whose insertion point to search for + * @param a the array in which to search + * @param base the index of the first element in the range + * @param len the length of the range; must be > 0 + * @param hint the index at which to begin the search, 0 <= hint < n. + * The closer hint is to the result, the faster this method will run. + * @param c the comparator used to order the range, and to search + * @return the int k, 0 <= k <= n such that a[b + k - 1] < key <= a[b + k], + * pretending that a[b - 1] is minus infinity and a[b + n] is infinity. + * In other words, key belongs at index b + k; or in other words, + * the first k elements of a should precede key, and the last n - k + * should follow it. + */ + private static int gallopLeft(int key, int[] a, int base, int len, int hint, + ExposedComparators.OrdinalComparator c) { + assert len > 0 && hint >= 0 && hint < len; + int lastOfs = 0; + int ofs = 1; + if (c.compare(key, a[base + hint]) > 0) { + // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs] + int maxOfs = len - hint; + while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) > 0) { + lastOfs = ofs; + ofs = (ofs << 1) + 1; + if (ofs <= 0) // int overflow + ofs = maxOfs; + } + if (ofs > maxOfs) + ofs = maxOfs; + + // Make offsets relative to base + lastOfs += hint; + ofs += hint; + } else { // key <= a[base + hint] + // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs] + final int maxOfs = hint + 1; + while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) <= 0) { + lastOfs = ofs; + ofs = (ofs << 1) + 1; + if (ofs <= 0) // int overflow + ofs = maxOfs; + } + if (ofs > maxOfs) + ofs = maxOfs; + + // Make offsets relative to base + int tmp = lastOfs; + lastOfs = hint - ofs; + ofs = hint - tmp; + } + assert -1 <= lastOfs && lastOfs < ofs && ofs <= len; + + /* + * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere + * to the right of lastOfs but no farther right than ofs. Do a binary + * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs]. + */ + lastOfs++; + while (lastOfs < ofs) { + int m = lastOfs + ((ofs - lastOfs) >>> 1); + + if (c.compare(key, a[base + m]) > 0) + lastOfs = m + 1; // a[base + m] < key + else + ofs = m; // key <= a[base + m] + } + assert lastOfs == ofs; // so a[base + ofs - 1] < key <= a[base + ofs] + return ofs; + } + + /** + * Like gallopLeft, except that if the range contains an element equal to + * key, gallopRight returns the index after the rightmost equal element. + * + * @param key the key whose insertion point to search for + * @param a the array in which to search + * @param base the index of the first element in the range + * @param len the length of the range; must be > 0 + * @param hint the index at which to begin the search, 0 <= hint < n. + * The closer hint is to the result, the faster this method will run. + * @param c the comparator used to order the range, and to search + * @return the int k, 0 <= k <= n such that a[b + k - 1] <= key < a[b + k] + */ + private static int gallopRight(int key, int[] a, int base, int len, + int hint, ExposedComparators.OrdinalComparator c) { + assert len > 0 && hint >= 0 && hint < len; + + int ofs = 1; + int lastOfs = 0; + if (c.compare(key, a[base + hint]) < 0) { + // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs] + int maxOfs = hint + 1; + while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) < 0) { + lastOfs = ofs; + ofs = (ofs << 1) + 1; + if (ofs <= 0) // int overflow + ofs = maxOfs; + } + if (ofs > maxOfs) + ofs = maxOfs; + + // Make offsets relative to b + int tmp = lastOfs; + lastOfs = hint - ofs; + ofs = hint - tmp; + } else { // a[b + hint] <= key + // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs] + int maxOfs = len - hint; + while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) >= 0) { + lastOfs = ofs; + ofs = (ofs << 1) + 1; + if (ofs <= 0) // int overflow + ofs = maxOfs; + } + if (ofs > maxOfs) + ofs = maxOfs; + + // Make offsets relative to b + lastOfs += hint; + ofs += hint; + } + assert -1 <= lastOfs && lastOfs < ofs && ofs <= len; + + /* + * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to + * the right of lastOfs but no farther right than ofs. Do a binary + * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs]. + */ + lastOfs++; + while (lastOfs < ofs) { + int m = lastOfs + ((ofs - lastOfs) >>> 1); + + if (c.compare(key, a[base + m]) < 0) + ofs = m; // key < a[b + m] + else + lastOfs = m + 1; // a[b + m] <= key + } + assert lastOfs == ofs; // so a[b + ofs - 1] <= key < a[b + ofs] + return ofs; + } + + /** + * Merges two adjacent runs in place, in a stable fashion. The first + * element of the first run must be greater than the first element of the + * second run (a[base1] > a[base2]), and the last element of the first run + * (a[base1 + len1-1]) must be greater than all elements of the second run. + * + * For performance, this method should be called only when len1 <= len2; + * its twin, mergeHi should be called if len1 >= len2. (Either method + * may be called if len1 == len2.) + * + * @param base1 index of first element in first run to be merged + * @param len1 length of first run to be merged (must be > 0) + * @param base2 index of first element in second run to be merged + * (must be aBase + aLen) + * @param len2 length of second run to be merged (must be > 0) + */ + private void mergeLo(int base1, int len1, int base2, int len2) { + assert len1 > 0 && len2 > 0 && base1 + len1 == base2; + + // Copy first run into temp array + int[] a = this.a; // For performance + int[] tmp = ensureCapacity(len1); + System.arraycopy(a, base1, tmp, 0, len1); + + int cursor1 = 0; // Indexes into tmp array + int cursor2 = base2; // Indexes int a + int dest = base1; // Indexes int a + + // Move first element of second run and deal with degenerate cases + a[dest++] = a[cursor2++]; + if (--len2 == 0) { + System.arraycopy(tmp, cursor1, a, dest, len1); + return; + } + if (len1 == 1) { + System.arraycopy(a, cursor2, a, dest, len2); + a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge + return; + } + + ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance + int minGallop = this.minGallop; // " " " " " + outer: + while (true) { + int count1 = 0; // Number of times in a row that first run won + int count2 = 0; // Number of times in a row that second run won + + /* + * Do the straightforward thing until (if ever) one run starts + * winning consistently. + */ + do { + assert len1 > 1 && len2 > 0; + if (c.compare(a[cursor2], tmp[cursor1]) < 0) { + a[dest++] = a[cursor2++]; + count2++; + count1 = 0; + if (--len2 == 0) + break outer; + } else { + a[dest++] = tmp[cursor1++]; + count1++; + count2 = 0; + if (--len1 == 1) + break outer; + } + } while ((count1 | count2) < minGallop); + + /* + * One run is winning so consistently that galloping may be a + * huge win. So try that, and continue galloping until (if ever) + * neither run appears to be winning consistently anymore. + */ + do { + assert len1 > 1 && len2 > 0; + count1 = gallopRight(a[cursor2], tmp, cursor1, len1, 0, c); + if (count1 != 0) { + System.arraycopy(tmp, cursor1, a, dest, count1); + dest += count1; + cursor1 += count1; + len1 -= count1; + if (len1 <= 1) // len1 == 1 || len1 == 0 + break outer; + } + a[dest++] = a[cursor2++]; + if (--len2 == 0) + break outer; + + count2 = gallopLeft(tmp[cursor1], a, cursor2, len2, 0, c); + if (count2 != 0) { + System.arraycopy(a, cursor2, a, dest, count2); + dest += count2; + cursor2 += count2; + len2 -= count2; + if (len2 == 0) + break outer; + } + a[dest++] = tmp[cursor1++]; + if (--len1 == 1) + break outer; + minGallop--; + } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP); + if (minGallop < 0) + minGallop = 0; + minGallop += 2; // Penalize for leaving gallop mode + } // End of "outer" loop + this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field + + if (len1 == 1) { + assert len2 > 0; + System.arraycopy(a, cursor2, a, dest, len2); + a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge + } else if (len1 == 0) { + throw new IllegalArgumentException( + "Comparison method violates its general contract!"); + } else { + assert len2 == 0; + assert len1 > 1; + System.arraycopy(tmp, cursor1, a, dest, len1); + } + } + + /** + * Like mergeLo, except that this method should be called only if + * len1 >= len2; mergeLo should be called if len1 <= len2. (Either method + * may be called if len1 == len2.) + * + * @param base1 index of first element in first run to be merged + * @param len1 length of first run to be merged (must be > 0) + * @param base2 index of first element in second run to be merged + * (must be aBase + aLen) + * @param len2 length of second run to be merged (must be > 0) + */ + private void mergeHi(int base1, int len1, int base2, int len2) { + assert len1 > 0 && len2 > 0 && base1 + len1 == base2; + + // Copy second run into temp array + int[] a = this.a; // For performance + int[] tmp = ensureCapacity(len2); + System.arraycopy(a, base2, tmp, 0, len2); + + int cursor1 = base1 + len1 - 1; // Indexes into a + int cursor2 = len2 - 1; // Indexes into tmp array + int dest = base2 + len2 - 1; // Indexes into a + + // Move last element of first run and deal with degenerate cases + a[dest--] = a[cursor1--]; + if (--len1 == 0) { + System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2); + return; + } + if (len2 == 1) { + dest -= len1; + cursor1 -= len1; + System.arraycopy(a, cursor1 + 1, a, dest + 1, len1); + a[dest] = tmp[cursor2]; + return; + } + + ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance + int minGallop = this.minGallop; // " " " " " + outer: + while (true) { + int count1 = 0; // Number of times in a row that first run won + int count2 = 0; // Number of times in a row that second run won + + /* + * Do the straightforward thing until (if ever) one run + * appears to win consistently. + */ + do { + assert len1 > 0 && len2 > 1; + if (c.compare(tmp[cursor2], a[cursor1]) < 0) { + a[dest--] = a[cursor1--]; + count1++; + count2 = 0; + if (--len1 == 0) + break outer; + } else { + a[dest--] = tmp[cursor2--]; + count2++; + count1 = 0; + if (--len2 == 1) + break outer; + } + } while ((count1 | count2) < minGallop); + + /* + * One run is winning so consistently that galloping may be a + * huge win. So try that, and continue galloping until (if ever) + * neither run appears to be winning consistently anymore. + */ + do { + assert len1 > 0 && len2 > 1; + count1 = len1 - gallopRight(tmp[cursor2], a, base1, len1, len1 - 1, c); + if (count1 != 0) { + dest -= count1; + cursor1 -= count1; + len1 -= count1; + System.arraycopy(a, cursor1 + 1, a, dest + 1, count1); + if (len1 == 0) + break outer; + } + a[dest--] = tmp[cursor2--]; + if (--len2 == 1) + break outer; + + count2 = len2 - gallopLeft(a[cursor1], tmp, 0, len2, len2 - 1, c); + if (count2 != 0) { + dest -= count2; + cursor2 -= count2; + len2 -= count2; + System.arraycopy(tmp, cursor2 + 1, a, dest + 1, count2); + if (len2 <= 1) // len2 == 1 || len2 == 0 + break outer; + } + a[dest--] = a[cursor1--]; + if (--len1 == 0) + break outer; + minGallop--; + } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP); + if (minGallop < 0) + minGallop = 0; + minGallop += 2; // Penalize for leaving gallop mode + } // End of "outer" loop + this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field + + if (len2 == 1) { + assert len1 > 0; + dest -= len1; + cursor1 -= len1; + System.arraycopy(a, cursor1 + 1, a, dest + 1, len1); + a[dest] = tmp[cursor2]; // Move first elt of run2 to front of merge + } else if (len2 == 0) { + throw new IllegalArgumentException( + "Comparison method violates its general contract!"); + } else { + assert len1 == 0; + assert len2 > 0; + System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2); + } + } + + /** + * Ensures that the external array tmp has at least the specified + * number of elements, increasing its size if necessary. The size + * increases exponentially to ensure amortized linear time complexity. + * + * @param minCapacity the minimum required capacity of the tmp array + * @return tmp, whether or not it grew + */ + private int[] ensureCapacity(int minCapacity) { + if (tmp.length < minCapacity) { + // Compute smallest power of 2 > minCapacity + int newSize = minCapacity; + newSize |= newSize >> 1; + newSize |= newSize >> 2; + newSize |= newSize >> 4; + newSize |= newSize >> 8; + newSize |= newSize >> 16; + newSize++; + + if (newSize < 0) // Not bloody likely! + newSize = minCapacity; + else + newSize = Math.min(newSize, a.length >>> 1); + + tmp = new int[newSize]; + } + return tmp; + } + + /** + * Checks that fromIndex and toIndex are in range, and throws an + * appropriate exception if they aren't. + * + * @param arrayLen the length of the array + * @param fromIndex the index of the first element of the range + * @param toIndex the index after the last element of the range + * @throws IllegalArgumentException if fromIndex > toIndex + * @throws ArrayIndexOutOfBoundsException if fromIndex < 0 + * or toIndex > arrayLen + */ + private static void rangeCheck(int arrayLen, int fromIndex, int toIndex) { + if (fromIndex > toIndex) + throw new IllegalArgumentException("fromIndex(" + fromIndex + + ") > toIndex(" + toIndex+")"); + if (fromIndex < 0) + throw new ArrayIndexOutOfBoundsException(fromIndex); + if (toIndex > arrayLen) + throw new ArrayIndexOutOfBoundsException(toIndex); + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalFacets.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalFacets.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalFacets.java (revision ) @@ -0,0 +1,322 @@ +package org.apache.lucene.search.exposed.facet; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.search.exposed.*; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.apache.lucene.util.packed.PackedInts; + +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; + +// TODO: Change this to LuceneTestCase but ensure Flex +public class TestHierarchicalFacets extends TestCase { + private ExposedHelper helper; + private ExposedCache cache; + + public TestHierarchicalFacets(String name) { + super(name); +// CodecProvider.setDefaultCodec("Standard"); + TestHierarchicalTermProvider.deleteIndex(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + cache = ExposedCache.getInstance(); + helper = new ExposedHelper(); + } + + @Override + + public void tearDown() throws Exception { + super.tearDown(); + cache.purgeAllCaches(); + helper.close(); + } + + + public static Test suite() { + return new TestSuite(TestHierarchicalFacets.class); + } + + public void testBasicIndexBuild() throws IOException { + TestHierarchicalTermProvider.createIndex(1000, 3, 4); + } + + public static final String HIERARCHICAL_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public static final String HIERARCHICAL_POP_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public static final String NON_HIERARCHICAL_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + //http://wiki.apache.org/solr/HierarchicalFaceting + public static final String HIERARCHICAL_COMPARATIVE_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + public static final String HIERARCHICAL_COMPARATIVE_SANE_REQUEST = + "\n" + + "\n" + + " even:true\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + + // http://sbdevel.wordpress.com/2010/10/05/fast-hierarchical-faceting/ + public int buildSampleIndex() throws IOException { + IndexWriter w = ExposedHelper.getWriter(); + ExposedHelper.addDocument(w, + ExposedHelper.ID + ":1", + ExposedHelper.ALL + ":" + ExposedHelper.ALL, + "even:true", + "deep:A/B/C", + "deep:D/E/F"); + ExposedHelper.addDocument(w, + ExposedHelper.ID + ":2", + ExposedHelper.ALL + ":" + ExposedHelper.ALL, + "even:false", + "deep:A/B/C", + "deep:A/B/J"); + ExposedHelper.addDocument(w, + ExposedHelper.ID + ":3", + ExposedHelper.ALL + ":" + ExposedHelper.ALL, + "even:true", + "deep:A", + "deep:D/E", + "deep:G/H/I"); + w.close(true); + return 3; + } + + //http://wiki.apache.org/solr/HierarchicalFaceting + public int buildComparativeIndex(int level2num) throws IOException { + IndexWriter w = ExposedHelper.getWriter(); + int docID = 0; + for (char c = 'A' ; c <= 'Z' ; c++) { + for (int i = 1 ; i <= level2num ; i++) { + ExposedHelper.addDocument(w, + "all:all", + "id:" + Integer.toString(docID++), + "level_s:" + Character.toString(c) + "/" + i, + "level1_s:" + Character.toString(c), + "level12_s:" + i); + } + } + w.close(true); + return docID; + } + + public void testReflection() throws XMLStreamException { + FacetRequest request = FacetRequest.parseXML(HIERARCHICAL_REQUEST); + System.out.println(request.toXML()); + } + + public void testMapOld() throws IOException { + final int DOCS = buildSampleIndex(); + + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + TermProvider basic = ExposedCache.getInstance().getProvider( + reader, "myGroup", Arrays.asList("deep"), null, + ExposedRequest.LUCENE_ORDER); + FacetMap map = new FacetMap(reader.maxDoc(), Arrays.asList(basic)); + for (int i = 0 ; i < reader.maxDoc() ; i++) { + System.out.print("Doc " + i + ":"); + BytesRef[] terms = map.getTermsForDocID(i); + for (BytesRef term: terms) { + System.out.print(" " + term.utf8ToString()); + } + System.out.println(""); + } + } + + public void testMap() throws IOException { + final int DOCS = 4; + TestHierarchicalTermProvider.createIndex(DOCS, 5, 4); + + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + TermProvider basic = ExposedCache.getInstance().getProvider( + reader, "myGroup", Arrays.asList("deep"), null, + ExposedRequest.LUCENE_ORDER); + FacetMap map = new FacetMap(reader.maxDoc(), Arrays.asList(basic)); + for (int i = 0 ; i < reader.maxDoc() ; i++) { + System.out.print("Doc " + i + ":"); + BytesRef[] terms = map.getTermsForDocID(i); + for (BytesRef term: terms) { + System.out.print(" " + term.utf8ToString()); + } + System.out.println(""); + } + } + + public void testOrderedControlled() throws Exception { +// TestHierarchicalTermProvider.createIndex(10, 5, 4); + buildSampleIndex(); + testRequest(HIERARCHICAL_REQUEST, 1); + } + + public void testPopularityControlled() throws Exception { + buildSampleIndex(); + testRequest(HIERARCHICAL_POP_REQUEST, 1); + } + + public void testOrderedWide() throws Exception { + TestHierarchicalTermProvider.createIndex(1000000, 5, 4); + ExposedSettings.debug = true; + testRequest(HIERARCHICAL_REQUEST, 5); + } + + public void testOrderedDeep() throws Exception { + TestHierarchicalTermProvider.createIndex(10000, 4, 15); + ExposedSettings.debug = true; + testRequest(HIERARCHICAL_REQUEST, 5); + } + + public void testPopularitySimple() throws Exception { + TestHierarchicalTermProvider.createIndex(10000, 5, 4); + ExposedSettings.debug = true; + testRequest(HIERARCHICAL_POP_REQUEST, 3); + } + + // http://wiki.apache.org/solr/HierarchicalFaceting + public void testComparative() throws Exception { + buildComparativeIndex(10000); + ExposedSettings.debug = true; + testRequest(HIERARCHICAL_COMPARATIVE_REQUEST, 5); + } + + public void testComparativeSane() throws Exception { + buildComparativeIndex(10000); + ExposedSettings.debug = true; + testRequest(HIERARCHICAL_COMPARATIVE_SANE_REQUEST, 5); + } + + public void testRequest(String requestXML, int runs) throws Exception { + //File location = new File("/home/te/projects/index10M10T4L"); + File location = ExposedHelper.INDEX_LOCATION; + + IndexReader reader = ExposedIOFactory.getReader(location); + IndexSearcher searcher = new IndexSearcher(reader); + QueryParser qp = new QueryParser( + Version.LUCENE_31, ExposedHelper.ALL, + new MockAnalyzer(new Random(), MockTokenizer.WHITESPACE, false)); + Query q = qp.parse(ExposedHelper.ALL); + TopScoreDocCollector sanityCollector = + TopScoreDocCollector.create(10, false); + searcher.search(q, sanityCollector); + assertEquals("The search for " + q + " should give the right hits", + reader.maxDoc(), sanityCollector.topDocs().totalHits); + long preMem = ExposedHelper.getMem(); + long facetStructureTime = System.currentTimeMillis(); + + CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2); + + FacetRequest request = FacetRequest.parseXML(requestXML); + CollectorPool collectorPool = poolFactory.acquire(reader, request); + facetStructureTime = System.currentTimeMillis() - facetStructureTime; + + TagCollector collector; + FacetResponse response = null; + String sQuery = null;//request.getQuery(); + for (int i = 0 ; i < runs ; i++) { + collector = collectorPool.acquire(sQuery); + long countStart = System.currentTimeMillis(); + if (collector.getQuery() == null) { // Fresh collector + searcher.search(q, collector); + if (i == 0) { + System.out.println(collector.toString(true)); + } +// collector.collectAllValid(reader); + long countTime = System.currentTimeMillis() - countStart; + collector.setCountTime(countTime); + } + response = collector.extractResult(request); + if (collector.getQuery() != null) { // Cached count + response.setCountingCached(true); + } + long totalTime = System.currentTimeMillis() - countStart; + response.setTotalTime(totalTime); + System.out.println("Collection and extraction #" + i + " for " + + collector.getHitCount() + " documents in " + + ExposedHelper.getTime(System.currentTimeMillis()-countStart)); + collectorPool.release(sQuery, collector); + } + System.out.println("Document count = " + reader.maxDoc()); + System.out.println("Hierarchical facet startup time = " + + ExposedHelper.getTime(facetStructureTime)); + System.out.println("Mem usage: preFacet=" + preMem + + " MB, postFacet=" + ExposedHelper.getMem() + " MB"); + if (response != null) { + long xmlTime = -System.currentTimeMillis(); + String r = response.toXML(); + xmlTime += System.currentTimeMillis(); + System.out.println("Generated XML with " + r.length() + " characters in " + + xmlTime + " ms"); + System.out.println(r.length() > 10000 ? r.substring(0, 10000) : r); + } + } +} \ No newline at end of file Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xsd =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xsd (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/request/FacetRequest.xsd (revision ) @@ -0,0 +1,163 @@ + + + + + XML Schema for Lucene exposed (LUCENE-2369) requests. + + + + + + + + + The query is the unmodified query used for the Lucene search to + provide faceting for. + order: count = the tags are sorted by occurrences. + Highest number comes first. + index = the tags sorted by index order (BytesRef term + natural order). + locale: the tags are sorted by the locale given in the + attribute "locale". + The default order is count. + locale: If sort is specified to "locale", this locale is used. + maxtags: The maximum number of tags to return for a facet. + -1 is unlimited and _not_ recommended. + The default is 20. + mincount: The minimum number of occurrences in order for a tag + to be part of the result. + The default is 0. + offset: Where to start extracting tags in the sorted list of tags. + Used for pagination. + The default is 0. + Note: This can be negative when used with "prefix" and + order != count. + prefix: The extraction starts at (the first tag that matches the + prefix) + offset. This cannot be used with count order. + The default is "", meaning the beginning of the sorted + tag list. + + + + + + + + + + + + + + + + + + + + + + + + + + hierarchical: If true, hierarchical tags are assumed. + Default is false. + levels: If hierarchical is true, this is the maximum number + of levels to expand where plain single-level faceting + is level 1. + Default is 5. + delimiter: If hierarchical is true, this states the delimiter + for the levels in the tags. + Default is '/'. + startpath: The starting path for the facet extraction. This is useful + for building a GUI where tags can be expanded. Note that + the starting path does not change the hierarchical + application of the subtags field, it just skips down to + the starting point. Also note that counts for the + intermediate levels on the path might be undefined due + to performance optimization. + Example: 'A/B/C/'. + This only has effect with hierarchical == true. + Default is no starting path. + + If the group is hierarchical and subtags is defined, the subtags + attributes are used from the root. This means that order can + be specified as group attribute as well as subtags. This makes + sense e.g. if the overall order should be location aware, but + the root-level tags should be sorted by count. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + suborder: base or count. Base means whatever order the provider for the + group delivers (normally unicode or locale-aware order). + Default is count for first level if facet sort is count, then + inherited. + mintotalcount: As mincount but includes the counts for subtags. + Default is 0. + Default values for all attributes other than suborder and mintotalcount + are inherited. + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedRequest.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedRequest.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedRequest.java (revision ) @@ -0,0 +1,233 @@ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.util.BytesRef; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A request defines how a TermProvider is to behave with regard to fields + * and sorting. + *

+ * A FacetGroup is the primary access point for sorting. A FacetGroup contains one or + * more Fields from one or more segments. + *

+ * Field is single field and single segment oriented. It is used internally by + * FacetGroup but can be used externally if the index is fully optimized and sorting + * is requested for a single field. + */ +public class ExposedRequest { + /** + * If the comparator key is set to the value "LUCENE", the order of the + * terms is Lucene's default order. + */ + public static final String LUCENE_ORDER = "LUCENE"; + public static final String FREE_ORDER = "FREE"; + + /** + * This signifies that the order of the terms does not matter to the caller. + * The normal use-cache is for count-based faceting. By specifying WHATEVER + * as the order, the TermProvider cache is free to return any element that + * is based on the right fields. + */ + public static final String WHATEVER_ORDER = "WHATEVER"; + + /** + * A group is a collections of fields that are to be treated as one. + * An example could be a group named "title" with the fields "lti", + * "sort_title" and "subtitle" as fields. The same comparator must be used + * for all fields in order for term ordering to be consistent for the group. + */ + public static class Group { + private final String name; + private final List fields; + private final Comparator comparator; + private final boolean reverse; + private String comparatorID; + + public Group(String name, List fields, + Comparator comparator, boolean reverse, + String comparatorID) { + this.name = name; + this.fields = fields; + this.comparator = comparator; + this.reverse = reverse; + this.comparatorID = comparatorID; + } + + /** + * Checks whether the given FacetGroup is equal to . The order of the contained + * fields is irrelevant, but the set of fields must be equal. + *

+ * The name of the group is not used while comparing. + * @param other the group to compare to. + * @return true if the two groups are equivalent.. + */ + // TODO: Fix hashcode to fulfill the equals vs. hashcode contract + public boolean equals(Group other) { + if (other.getFields().size() != getFields().size() || + !getComparatorID().equals(other.getComparatorID())) { + return false; + } + + fieldLoop: + for (Field otherField: other.getFields()) { + for (Field thisField: getFields()) { + if (thisField.equals(otherField)) { + continue fieldLoop; + } + } + return false; + } + return true; + } + + /** + * Similar to {@link #equals(Group)} but if the comparatorID for the other + * group is {@link #FREE_ORDER}, it matches any comparatorID this group has. + * @param other the group to check with. + * @return true if this group can deliver data for the other group. + */ + public boolean worksfor(Group other) { + if (other.getFields().size() != getFields().size()) { + return false; + } + if (!FREE_ORDER.equals(other.getComparatorID()) && + !getComparatorID().equals(other.getComparatorID())) { + return false; + } + + fieldLoop: + for (Field otherField: other.getFields()) { + for (Field thisField: getFields()) { + if (thisField.worksfor(otherField)) { + continue fieldLoop; + } + } + return false; + } + return true; + } + + public String getName() { + return name; + } + public List getFields() { + return fields; + } + public Comparator getComparator() { + return comparator; + } + public boolean isReverse() { + return reverse; + } + public String getComparatorID() { + return comparatorID; + } + + public List getFieldNames() { + List fieldNames = new ArrayList(fields.size()); + for (Field field: fields) { + fieldNames.add(field.getField()); + } + return fieldNames; + } + + /** + * If the comparatorID is {@link #FREE_ORDER} we set it to + * {@link #LUCENE_ORDER}; + */ + void normalizeComparatorIDs() { + if (!FREE_ORDER.equals(getComparatorID())) { + return; + } + comparatorID = FREE_ORDER; + for (Field field: fields) { + field.comparatorID = FREE_ORDER; + } + } + } + + public static class Field { + private String field; + private Comparator comparator; + private boolean reverse; + private String comparatorID; + + public Field( + String field, Comparator comparator, boolean reverse, + String comparatorID) { + this.field = field; + this.comparator = comparator; + this.reverse = reverse; + this.comparatorID = comparatorID; + } + + /** + * Checks whether the given object is also a Field and if so whether the + * field and comparatorIDs are equal. + * @param obj candidate for equality. + * @return true if the given object is equivalent to this. + */ + public boolean equals(Object obj) { + if (!(obj instanceof Field)) { + return false; + } + Field other = (Field)obj; + return field.equals(other.field) + && comparatorID.equals(other.comparatorID); + } + + /** + * Similar to {@link #equals} but if the comparatorID for the other + * field is {@link #FREE_ORDER}, it matches any comparatorID this field has. + * @param other the field to check with. + * @return true if this group can deliver data for the other field. + **/ + public boolean worksfor(Field other) { + return field.equals(other.field) + && (FREE_ORDER.equals(other.comparatorID) + ||comparatorID.equals(other.comparatorID)); + } + + public String getField() { + return field; + } + public Comparator getComparator() { + return comparator; + } + + public boolean isReverse() { + return reverse; + } + + public String getComparatorID() { + return comparatorID; + } + + public boolean equals(Field otherField) { + return getField().equals(otherField.getField()) && + getComparatorID().equals(otherField.getComparatorID()); + } + + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/OrdinalTermsEnum.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/OrdinalTermsEnum.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/OrdinalTermsEnum.java (revision ) @@ -0,0 +1,222 @@ +package org.apache.lucene.search.exposed; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.*; +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** + * Wrapper for a TermsEnum that guarantees that the methods + * {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)} are + * implemented. + *

+ * Important: the {@link #ord} is only consistent when the last position + * changing method used was {@link #seekExact(long)} or {@link #next()}. + *

+ * The wrapper works by keeping a TermState for every X ordinal and thus allows + * for a flexible space/speed trade off. + */ +// TODO: Measure memory impact as it seems horribly wasteful to keep term+state +// TODO: Make an option for only storing BytesRef +// TODO: Use ByteBlockPool or similar to hold BytesRefs (see BytesRefHash) +// TODO: Merge with Solr's NumberedTermsEnum +public class OrdinalTermsEnum extends TermsEnum { + public final TermsEnum inner; + public final int divider; + public final List marks; + public final List terms; + public final long termCount; + + private boolean ordOK = true; + private long ordinal = 0; + + /** + * A hack that creates a TermsEnum and tries to access the ordinal for the + * first term. If an exception is thrown, the TermsEnum is wrapped as an + * OrdinalTermsEnum. If no exception is thrown, the TermsEnum is reset and + * returned directly. + * @param reader the reader to request the TermsEnum from. + * @param field the field to request the TermsEnum for. + * @param divider keep a TermState for every X terms if an OrdinalTermsEnum + * is created. + * @return a plain TermsEnum if ordinal access is provided for the given field + * by the given reader, else an OrdinalTermsEnum. If no TermsEnum can + * be requested at all, null is returned. + * @throws java.io.IOException if the index could not be accessed. + */ + public static TermsEnum createEnum( + IndexReader reader, String field, int divider) throws IOException { + Terms terms = reader.fields().terms(field); + if (terms == null) { + return null; + } + TermsEnum inner = terms.iterator(); + if (inner.next() == null) { + return null; // No terms + } + try { + inner.ord(); + } catch (UnsupportedOperationException e) { + // No ordinal seeking, so we make our own + return new OrdinalTermsEnum(inner, divider); + } + return inner; + } + + /** + * @param inner the TermsEnum to use for all calls except ord() and + * seek(long). The TermsEnum must be positioned at the first + * term, which basically means that next() must have been + * called once. + * @param divider keep a TermState for every X terms. + * @throws java.io.IOException if it was not possible to extract TermStates. + */ + public OrdinalTermsEnum(TermsEnum inner, int divider) throws IOException { + this.inner = inner; + this.divider = divider; + marks = new ArrayList(); + terms = new ArrayList(); + long count = 0; + while (true) { + //System.out.println(count + ": " + inner.term().utf8ToString()); + if (count % divider == 0) { + marks.add(inner.termState()); + terms.add((BytesRef) inner.term().clone()); + } + count++; + if (inner.next() == null) { + break; + } + } + inner.seekExact(terms.get(0), marks.get(0)); + termCount = count; + } + + /** + * @return the number of unique terms in this enum. + */ + public long getTermCount() { + return termCount; + } + + @Override + public long ord() throws IOException { + if (!ordOK) { + throw new IllegalStateException( + "ord() can only be called when the last position-changing call was to " + + "seek(long) or next()"); + } + return ordinal; + } + + @Override + public void seekExact(long ord) throws IOException { + ordOK = true; + if (ordinal > ord || ord > ordinal + divider) { // We're outside the block + int pos = (int)(ord / divider); + inner.seekExact(terms.get(pos), marks.get(pos)); +// System.out.println("*** " + br.utf8ToString() + " " + ord); + ordinal = ord / divider * divider; + } + + while (ordinal < ord) { + if (inner.next() == null) { + throw new IOException( + "Ordinal " + ord + " exceeded the term iterators capacity"); + } + ordinal++; + } + } + + @Override + public BytesRef next() throws IOException { + BytesRef next = inner.next(); + if (next != null) { + ordinal++; + } + return next; + } + + /* Direct delegations */ + + @Override + public AttributeSource attributes() { + return inner.attributes(); + } + + @Override + public boolean seekExact(BytesRef text, boolean useCache) throws IOException { + ordOK = false; + return inner.seekExact(text, useCache); + } + + @Override + public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException { + ordOK = false; + return inner.seekCeil(text, useCache); + } + + @Override + public void seekExact(BytesRef term, TermState state) throws IOException { + ordOK = false; + inner.seekExact(term, state); + } + + @Override + public BytesRef term() throws IOException { + return inner.term(); + } + + @Override + public int docFreq() throws IOException { + return inner.docFreq(); + } + + @Override + public long totalTermFreq() throws IOException { + return inner.totalTermFreq(); + } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + return inner.docs(skipDocs, reuse); + } + + @Override + public DocsAndPositionsEnum docsAndPositions( + Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + return inner.docsAndPositions(skipDocs, reuse); + } + + @Override + public TermState termState() throws IOException { + return inner.termState(); + } + + @Override + public Comparator getComparator() throws IOException { + return inner.getComparator(); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java (revision ) @@ -0,0 +1,285 @@ +package org.apache.lucene.search.exposed.poc; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.search.exposed.ExposedFieldComparatorSource; +import org.apache.lucene.search.exposed.ExposedIOFactory; +import org.apache.lucene.search.exposed.ExposedSettings; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Locale; +import java.util.Random; + +public class ExposedPOC { + private static final double MAX_HITS = 20; + + public static void main(String[] args) + throws IOException, InterruptedException, ParseException { + if (args.length == 2 && "optimize".equals(args[0])) { + optimize(new File(args[1])); + return; + } + if (args.length != 6) { + System.err.println("Need 6 arguments, got " + args.length + "\n"); + usage(); + return; + } + String method = args[0]; + File location = new File(args[1]); + String field = args[2]; + Locale locale = + "null".equalsIgnoreCase(args[3]) ? null : new Locale(args[3]); + String defaultField = args[4]; + ExposedSettings.PRIORITY priority = + ExposedSettings.PRIORITY.fromString(args[5]); + try { + shell(method, location, field, locale, defaultField, priority); + } catch (Exception e) { + //noinspection CallToPrintStackTrace + e.printStackTrace(); + usage(); + } + } + + private static void optimize(File location) throws IOException { + System.out.println("Optimizing " + location + "..."); + long startTimeOptimize = System.nanoTime(); + IndexWriter writer = ExposedIOFactory.getWriter(location); + writer.optimize(1); + System.out.println("Optimized index in " + nsToString( + System.nanoTime() - startTimeOptimize)); + writer.close(true); + } + + private static void shell( + String method, File location, String field, Locale locale, + String defaultField, ExposedSettings.PRIORITY priority) + throws IOException, InterruptedException, org.apache.lucene.queryparser.classic.ParseException { + System.out.println(String.format( + "Testing sorted search for index at '%s' with sort on field %s with " + + "locale %s, using sort-method %s, priority %s. Heap: %s", + location, field, locale, method, priority, getHeap())); + ExposedSettings.priority = priority; + + org.apache.lucene.index.IndexReader reader = ExposedIOFactory.getReader(location); + System.out.println(String.format( + "Opened index of size %s from %s. the indes has %d documents and %s" + + " deletions. Heap: %s", + readableSize(calculateSize(location)), location, + reader.maxDoc(), + reader.hasDeletions() ? "some" : " no", + getHeap())); + +/* System.out.println(String.format( + "Creating %s Sort for field %s with locale %s... Heap: %s", + method, field, locale, getHeap())); + */ + long startTimeSort = System.nanoTime(); + Sort sort; + if ("exposed".equals(method) || "expose".equals(method)) { + ExposedFieldComparatorSource exposedFCS = + new ExposedFieldComparatorSource(reader, locale); + sort = new Sort(new SortField(field, exposedFCS)); + } else if ("default".equals(method)) { + if (locale == null) { + sort = new Sort(new SortField(field, SortField.Type.STRING)); + } else { + throw new UnsupportedOperationException( + "native sort by locale not supported in Lucene 4 trunk"); +/* + sort = locale == null ? + new Sort(new SortField(field, SortField.STRING)) : + new Sort(new SortField(field, locale)); + + */ + } + } else { + throw new IllegalArgumentException( + "The sort method " + method + " is unsupported"); + } + long sortTime = System.nanoTime() - startTimeSort; + + System.out.println(String.format( + "Created %s Sort for field %s in %s. Heap: %s", + method, field, nsToString(sortTime), getHeap())); + + IndexSearcher searcher = new IndexSearcher(reader); + + System.out.println(String.format( + "\nFinished initializing %s structures for field %s.\n" + + "Write standard Lucene queries to experiment with sorting speed.\n" + + "The StandardAnalyser will be used and the default field is %s.\n" + + "Finish with 'EXIT'.", method, field, defaultField)); + String query; + BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); + org.apache.lucene.queryparser.classic.QueryParser qp = + new org.apache.lucene.queryparser.classic.QueryParser( + Version.LUCENE_40, defaultField, new MockAnalyzer(new Random())); + + boolean first = true; + while (true) { + if (first) { + System.out.print("\nQuery (" + method + " sort, first search might take " + + "a while): "); + first = false; + } else { + System.out.print("Query (" + method + " sort): "); + } + query = br.readLine(); + if ("".equals(query)) { + continue; + } + if ("EXIT".equals(query)) { + break; + } + try { + long startTimeQuery = System.nanoTime(); + Query q = qp.parse(query); + long queryTime = System.nanoTime() - startTimeQuery; + + long startTimeSearch = System.nanoTime(); +// TopFieldDocs topDocs = searcher.search( +// q.weight(searcher), null, 20, sort, true); + TopFieldDocs topDocs = searcher.search(q, 20, sort); + long searchTime = System.nanoTime() - startTimeSearch; + System.out.println(String.format( + "The search for '%s' got %d hits in %s (+ %s for query parsing). " + + "Showing %d hits.", + query, topDocs.totalHits, + nsToString(searchTime), + nsToString(queryTime), + (int)Math.min(topDocs.totalHits, MAX_HITS))); + long startTimeDisplay = System.nanoTime(); + for (int i = 0 ; i < Math.min(topDocs.totalHits, MAX_HITS) ; i++) { + int docID = topDocs.scoreDocs[i].doc; + System.out.println(String.format( + "Hit #%d was doc #%d with %s:%s", + i, docID, field, + ((BytesRef)((FieldDoc)topDocs.scoreDocs[i]).fields[0]). + utf8ToString())); + } + System.out.print("Displaying the search result took " + + nsToString( + System.nanoTime() - startTimeDisplay) + ". "); + System.out.println("Heap: " + getHeap()); + } catch (Exception e) { + //noinspection CallToPrintStackTrace + e.printStackTrace(); + } + } + System.out.println("\nThank you for playing. Please come back."); + } + + private static void usage() { + System.out.println( + "Usage: ExposedPOC exposed|default " + + " \n" + + "exposed: Uses the expose sorter\n" + + "default: Uses the default sorter\n" + + ": The location of an optimized Lucene index\n" + + ": The field to use for sorting\n" + + ": The locale to use for sorting. If null is " + + "specified, natural term order is used\n" + + ": The field to search when no explicit field is " + + "given\n" + + ": Either speed or memory. Only relevant for " + + "exposed\n" + + "\n" + + "Example:\n" + + "ExposedPOC expose /mnt/bulk/40GB_index author da freetext" + + "\n" + + "If the index is is to be optimized, it can be done with\n" + + "ExposedPOC optimize \n" + + "\n" + + "Note: Heap-size is queried after a call to System.gc()\n" + + "Note: This is a proof of concept. Expect glitches!" + ); + } + + static String getHeap() throws InterruptedException { + String b = "Before GC: " + getHeapDirect(); + for (int i = 0 ; i < 1 ; i++) { + System.gc(); + Thread.sleep(10); + } + return b + ", after GC: " + getHeapDirect(); + } + + private static String getHeapDirect() { + return readableSize(Runtime.getRuntime().totalMemory() + - Runtime.getRuntime().freeMemory()) + "/" + + readableSize(Runtime.getRuntime().totalMemory()); + } + + static String readableSize(long size) { + return size > 2 * 1048576 ? + size / 1048576 + "MB" : + size > 2 * 1024 ? + size / 1024 + "KB" : + size + "bytes"; + } + + static long calculateSize(File file) { + long size = 0; + if (file.isDirectory()) { + for (File sub: file.listFiles()) { + size += calculateSize(sub); + } + } else { + size += file.length(); + } + return size; + } + + static String measureSortTime(final PackedInts.Reader orderedDocs) { + Integer[] allDocIDS = new Integer[orderedDocs.size()]; + for (int i = 0 ; i < allDocIDS.length ; i++) { + allDocIDS[i] = i; + } + long startTimeSort = System.nanoTime(); + Arrays.sort(allDocIDS, new Comparator() { + public int compare(Integer o1, Integer o2) { + return (int)(orderedDocs.get(o1) - orderedDocs.get(o2)); + } + }); + return nsToString( + System.nanoTime() - startTimeSort); + } + + static long footprint(PackedInts.Reader values) { + return values.getBitsPerValue() * values.size() / 8; + } + + static String nsToString(long time) { + return time > 10L * 1000 * 1000000 ? + minutes(time) + " min" : +// time > 2 * 1000000 ? + time / 1000000 + "ms";// : +// time + "ns"; + } + + private static String minutes(long num) { + long min = num / 60 / 1000 / 1000000; + long left = num - (min * 60 * 1000 * 1000000); + long sec = left / 1000 / 1000000; + String s = Long.toString(sec); + while (s.length() < 2) { + s = "0" + s; + } + return min + ":" + s; + } + + +} \ No newline at end of file Index: lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalTermProvider.java =================================================================== --- lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalTermProvider.java (revision ) +++ lucene/contrib/exposed/src/test/org/apache/lucene/search/exposed/facet/TestHierarchicalTermProvider.java (revision ) @@ -0,0 +1,212 @@ +package org.apache.lucene.search.exposed.facet; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.*; +import org.apache.lucene.search.exposed.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.regex.Pattern; + +// TODO: Change this to LuceneTestCase but ensure Flex +public class TestHierarchicalTermProvider extends TestCase { + public static final String HIERARCHICAL = "deep"; + + @Override + public void setUp() throws Exception { + super.setUp(); +// CodecProvider.setDefaultCodec("Standard"); + deleteIndex(); + } + public static void deleteIndex() { + if (ExposedHelper.INDEX_LOCATION.exists()) { + for (File file: ExposedHelper.INDEX_LOCATION.listFiles()) { + file.delete(); + } + ExposedHelper.INDEX_LOCATION.delete(); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + public static Test suite() { + return new TestSuite(TestHierarchicalTermProvider.class); + } + + public void testBasicTermBuildDump() throws IOException { + createIndex(1000, 3, 4); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + dumpField(reader, HIERARCHICAL, 10); + } + + public void testSplit() { + final String TAG = "A/B/C"; + final int EXPECTED_PARTS = 3; + final String REGEXP = "/"; + Pattern pattern = Pattern.compile(REGEXP); + assertEquals("The tag '" + TAG + "' should be split correctly by '" + + REGEXP + "'", EXPECTED_PARTS, pattern.split(TAG).length); + } + + public void testAugmentationDump() throws IOException { + final String REGEXP = "/"; + final int SIZE = 100; + final int MAX_DUMP = 10; + + createIndex(SIZE, 3, 4); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + TermProvider basic = ExposedCache.getInstance().getProvider( + reader, "myGroup", Arrays.asList(HIERARCHICAL), null, + ExposedRequest.LUCENE_ORDER); + HierarchicalTermProvider augmented = + new HierarchicalTermProvider(basic, REGEXP); + PackedInts.Reader aOrder = augmented.getOrderedOrdinals(); + for (int i = 0 ; i < aOrder.size() && i < MAX_DUMP ; i++) { + System.out.println("Tag #" + i + + ": L=" + augmented.getLevel(i) + + ", P=" + augmented.getPreviousMatchingLevel(i) + + " " + augmented.getOrderedTerm(i).utf8ToString()); + } + } + + public void testAugmentationBuildTime() throws IOException { + final String REGEXP = "/"; + final int[] SIZES = new int[]{100, 10000, 100000}; + final int RUNS = 3; + for (int size: SIZES) { + deleteIndex(); + long refs = createIndex(size, 5, 5); + IndexReader reader = + ExposedIOFactory.getReader(ExposedHelper.INDEX_LOCATION); + for (int i = 0 ; i < RUNS ; i++) { + ExposedCache.getInstance().purgeAllCaches(); + long basicTime = -System.currentTimeMillis(); + TermProvider basic = ExposedCache.getInstance().getProvider( + reader, "myGroup", Arrays.asList(HIERARCHICAL), null, + ExposedRequest.LUCENE_ORDER); + basicTime += System.currentTimeMillis(); + long buildTime = -System.currentTimeMillis(); + HierarchicalTermProvider augmented = + new HierarchicalTermProvider(basic, REGEXP); + buildTime += System.currentTimeMillis(); + System.out.println("Standard tp build for " + size + + " documents with a " + + "total of " + augmented.getOrderedOrdinals().size() + " tags " + + "referred " + + refs + " times: " + basicTime + " ms, " + + "augmented extra time: " + buildTime + " ms"); + } + } + } + + private void dumpField(IndexReader reader, String field, int tags) + throws IOException { + System.out.println("Dumping a maximum of " + tags + " from field " + field); + int count = 0; + IndexReader[] readers = reader.getSequentialSubReaders() == null ? + new IndexReader[]{reader} : reader.getSequentialSubReaders(); + DocsEnum docsEnum = null; + for (IndexReader r: readers) { + Terms mTerms = r.terms(field); + if (mTerms == null) { + continue; + } + assertNotNull("There should be terms for field " + field, mTerms); + TermsEnum terms = mTerms.iterator(); + while (terms.next() != null) { + System.out.print(terms.term().utf8ToString() + ":"); + docsEnum = terms.docs(r.getLiveDocs(), docsEnum); + while (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) { + System.out.print(" " + docsEnum.docID()); + } + System.out.println(""); + if (++count == tags) { + return; + } + } + } + } + + public static final String TAGS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + public static long createIndex(int docCount, int maxTagsPerLevel, int maxLevel) + throws IOException { + long startTime = System.nanoTime(); + long references = 0; + File location = ExposedHelper.INDEX_LOCATION; + Random random = new Random(87); + + + IndexWriter writer = ExposedIOFactory.getWriter(location); + + int every = docCount > 100 ? docCount / 100 : 1; + int next = every; + for (int docID = 0 ; docID < docCount ; docID++) { + if (docID == next) { + System.out.print("."); + next += every; + } + Document doc = new Document(); + + int levels = random.nextInt(maxLevel+1); + references += addHierarchicalTags( + doc, docID, random, levels, "", maxTagsPerLevel, false); + + doc.add(new Field(ExposedHelper.ID, ExposedHelper.ID_FORMAT.format(docID), + Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(ExposedHelper.EVEN, docID % 2 == 0 ? "true" : "false", + Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new Field(ExposedHelper.ALL, ExposedHelper.ALL, + Field.Store.YES, Field.Index.NOT_ANALYZED)); + writer.addDocument(doc); + } + writer.close(); + System.out.println(""); + System.out.println(String.format( + "Created %d document index with " + references + + " tag references in %sms at %s", docCount, + (System.nanoTime() - startTime) / 1000000, location.getAbsolutePath())); + return references; + } + + private static long addHierarchicalTags( + Document doc, int docID, Random random, int levelsLeft, String prefix, + int maxTagsPerLevel, boolean addDocID) { + long references = 0; + if (levelsLeft == 0) { + return references; + } + int tags = random.nextInt(maxTagsPerLevel+1); + if (tags == 0) { + tags = 1; + levelsLeft = 1; + } + String docIDAdder = addDocID ? "_" + docID : ""; + for (int i = 0 ; i < tags ; i++) { + String tag = levelsLeft == 1 ? + TAGS.charAt(random.nextInt(TAGS.length())) + docIDAdder : + "" + TAGS.charAt(random.nextInt(TAGS.length())); + if (levelsLeft == 1 || random.nextInt(10) == 0) { + doc.add(new Field(HIERARCHICAL, prefix + tag, + Field.Store.NO, Field.Index.NOT_ANALYZED)); + references++; + } + references += addHierarchicalTags(doc, docID, random, levelsLeft-1, + prefix + tag + "/", maxTagsPerLevel, addDocID); + } + return references; + } + +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/HierarchicalTermProvider.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/HierarchicalTermProvider.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/HierarchicalTermProvider.java (revision ) @@ -0,0 +1,248 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.exposed.ExposedSettings; +import org.apache.lucene.search.exposed.ExposedTuple; +import org.apache.lucene.search.exposed.TermProvider; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.GrowingMutable; +import org.apache.lucene.util.packed.PackedInts; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; +import java.util.regex.Pattern; + +/** + * Augments the hierarchical tag ordered entries from another TermProvider with + * level (A=1, A/B=2, A/B/C=3) and previous level match (A, A/B/C, D/E has 0, 1 + * and 0 respectively as there is no previous tag for A, "A" matches level 1 of + * "A/B/C" and "D/E" has no previous match at any level). + *

+ * The augmentation is used when doing hierarchical faceting. The memory + * overhead is {@code 2 * log2(maxdepht+1) bits}. Examples: + * 10 million tags with a maximum depth of 7 takes up 7 MB of heap. + * 50 million tags with a maximum depth of 30 takes up 60 MB of heap + * @see Fast, light, n-level hierarchical faceting. + */ +// TODO: Consider optimizing the cases where log2(maxlevel+1) <= 4 and <= 8 +// by backing with byte[] and [short] instead pf PackedInts. +// TODO: Consider optimizing build with split on byte(s) +public class HierarchicalTermProvider implements TermProvider { + private final TermProvider source; + private final PackedInts.Reader levels; + private final PackedInts.Reader pLevels; + private final String splitRegexp; + private final Pattern splitPattern; + + /** + * Calculates levels and previous level match for all ordered terms in the + * given provider. The HierarchicalTermProvider is ready for use after + * construction. + * Note that the caller is responsible for any escaping mechanism and for + * ensuring that the splitRegexp works consistently with character escaping. + * @param source a plain TermProvider containing hierarchical tags. + * @param splitRegexp the expression for splitting the tags from source into + * their level-components. If the tags is of the form + * A/B/C, the regexp would be "/". + * @throws java.io.IOException if the source could not be accessed. + */ + public HierarchicalTermProvider(TermProvider source, String splitRegexp) + throws IOException { + long buildTime = -System.currentTimeMillis(); + this.source = source; + this.splitRegexp = splitRegexp; + splitPattern = Pattern.compile(splitRegexp); + Pair lInfo = getLevels(); + levels = lInfo.getVal1(); + pLevels = lInfo.getVal2(); + buildTime += System.currentTimeMillis(); + if (ExposedSettings.debug) { + System.out.println("Extracted Hierarchical information from source with " + + source.getUniqueTermCount() + " unique terms in " + + buildTime + "ms"); + } + } + + /** + * Helper method for extracting the right tags. See the article that is linked + * in the class documentation for details. + * @param indirect an indirect for the underlying TermProvider. + * @param level the level to match against. + * @return true if {@code levels.get(indirect) >= level && + * pLevels.get(indirect) < level}. + */ + public boolean matchLevel(int indirect, int level) { + //return levels.get(indirect) >= level && pLevels.get(indirect) < level; + return levels.get(indirect) >= level && pLevels.get(indirect) >= level-1; + } + + /** + * @param indirect an indirect for the underlying TermProvider. + * @return the level for the given resolved tag. 'A' -> 1, 'A/B' -> 2 etc. + */ + public int getLevel(int indirect) { + return (int)levels.get(indirect); + } + + /** + * @param indirect an indirect for the underlying TermProvider. + * @return the previous match level for the given resolved tag. + */ + public int getPreviousMatchingLevel(int indirect) { + return (int)pLevels.get(indirect); + } + + private Pair getLevels() throws + IOException { + PackedInts.Reader ordered = source.getOrderedOrdinals(); + // FIXME: GrowingMutable should grow up to upper limit for bitsPerValue + final GrowingMutable levels = new GrowingMutable( + 0, ordered.size(), 0, 1, true); + final GrowingMutable pLevels = new GrowingMutable( + 0, ordered.size(), 0, 1, true); + String[] previous = new String[0]; + // TODO: Consider speeding up by sorting indirect chunks for seq. access + // TODO: Consider using StringTokenizer or custom split + long splitTime = 0; + for (int index = 0 ; index < ordered.size() ; index++) { + splitTime -= System.nanoTime(); + final String[] current = splitPattern.split( + source.getOrderedTerm(index).utf8ToString()); + splitTime += System.nanoTime(); + int pLevel = 0; + for (int level = 0 ; + level < current.length && level < previous.length ; + level++) { + if (current[level].equals(previous[level])) { + pLevel = level+1; + } + } + levels.set(index, current.length); + pLevels.set(index, pLevel); + previous = current; + } +/* System.out.println("Spend " + splitTime / 1000000 + " ms on " + + ordered.size() + " splits: " + splitTime / ordered.size() + + " ns/split");*/ + return new Pair( + reduce(levels), reduce(pLevels)); + } + + private PackedInts.Reader reduce(GrowingMutable grower) { + PackedInts.Mutable reduced = PackedInts.getMutable( + grower.size(), grower.getBitsPerValue()); + for (int i = 0 ; i < grower.size() ; i++) { + reduced.set(i, grower.get(i)); + } + return reduced; + } + + private final class Pair { + private final S val1; + private final T val2; + private Pair(S val1, T val2) { + this.val1 = val1; + this.val2 = val2; + } + public S getVal1() { + return val1; + } + public T getVal2() { + return val2; + } + } + + /* Plain delegations */ + + public int getNearestTermIndirect(BytesRef key) throws IOException { + return source.getNearestTermIndirect(key); + } + + public int getNearestTermIndirect( + BytesRef key, int startTermPos, int endTermPos) throws IOException { + return source.getNearestTermIndirect(key, startTermPos, endTermPos); + } + + public Comparator getComparator() { + return source.getComparator(); + } + + public String getComparatorID() { + return source.getComparatorID(); + } + + public String getDesignation() { + return source.getDesignation(); + } + + public String getField(long ordinal) throws IOException { + return source.getField(ordinal); + } + + public BytesRef getTerm(long ordinal) throws IOException { + return source.getTerm(ordinal); + } + + public String getOrderedField(long indirect) throws IOException { + return source.getOrderedField(indirect); + } + + public BytesRef getOrderedTerm(long indirect) throws IOException { + return source.getOrderedTerm(indirect); + } + + public long getUniqueTermCount() throws IOException { + return source.getUniqueTermCount(); + } + + public long getOrdinalTermCount() throws IOException { + return source.getOrdinalTermCount(); + } + + public long getMaxDoc() { + return source.getMaxDoc(); + } + + public IndexReader getReader() { + return source.getReader(); + } + + public int getReaderHash() { + return source.getReaderHash(); + } + + public int getRecursiveHash() { + return source.getRecursiveHash(); + } + + public PackedInts.Reader getOrderedOrdinals() throws IOException { + return source.getOrderedOrdinals(); + } + + public PackedInts.Reader getDocToSingleIndirect() throws IOException { + return source.getDocToSingleIndirect(); + } + + public Iterator getIterator( + boolean collectDocIDs) throws IOException { + return source.getIterator(collectDocIDs); + } + + public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException { + return source.getDocsEnum(ordinal, reuse); + } + + public void transitiveReleaseCaches(int level, boolean keepRoot) { + source.transitiveReleaseCaches(level, keepRoot); + } + + public int getDocIDBase() { + return source.getDocIDBase(); + } + + public void setDocIDBase(int base) { + source.setDocIDBase(base); + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ParseHelper.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ParseHelper.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/facet/ParseHelper.java (revision ) @@ -0,0 +1,43 @@ +package org.apache.lucene.search.exposed.facet; + +import org.apache.lucene.search.exposed.ExposedComparators; +import org.apache.lucene.search.exposed.facet.request.FacetRequest; +import org.apache.lucene.util.BytesRef; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import com.ibm.icu.text.Collator; +import java.util.Comparator; +import java.util.Locale; + +public class ParseHelper { + public static int getInteger(String request, String attribute, String value) + throws XMLStreamException { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new XMLStreamException("The integer for " + attribute + " was '" + + value + "' which is not valid. Full request: " + request); + } + } + + public static boolean atStart(XMLStreamReader reader, String tag) { + return reader.getEventType() == XMLStreamReader.START_ELEMENT + && tag.equals(reader.getLocalName()) + && FacetRequest.NAMESPACE.equals(reader.getNamespaceURI()); + } + + public static boolean atEnd(XMLStreamReader reader, String tag) { + return reader.getEventType() == XMLStreamReader.END_ELEMENT + && tag.equals(reader.getLocalName()) + && FacetRequest.NAMESPACE.equals(reader.getNamespaceURI()); + } + + public static Comparator createComparator(String locale) { + if (locale == null || "".equals(locale)) { + return ExposedComparators.collatorToBytesRef(null); + } + return ExposedComparators.collatorToBytesRef( + Collator.getInstance(new Locale(locale))); + } +} Index: solr/contrib/exposed/facet_samples.tcl =================================================================== --- solr/contrib/exposed/facet_samples.tcl (revision ) +++ solr/contrib/exposed/facet_samples.tcl (revision ) @@ -0,0 +1,193 @@ +#!/usr/bin/tclsh +# $Id: $ + +# +# Generates sample data for hierarchical faceting tests. +# Used primarily for testing SOLR-64, SOLR-792 and SOLR-2412 +# +# Note: SOLR-64 is single-path only so this can only be compared +# to SOLR-792 and SOLR-2412 if all elements are 1. If all elements +# are 1 and -64 is specified, +# + +proc usage {} { + puts "" + puts "Usage: facet_samples.tcl <-64> docs elements* <-u uniques*>" + puts "" + puts "-64 Special switch for generating output meant for SOLR-64." + puts " Setting this to true adds the field 'levels_h' to the output" + puts "docs: The number of documents to generate facet values for" + puts "elements: The number of elements at a given depth in the path for any document" + puts "uniques: The number of unique elements at a given depth" + puts " If no uniques are given, all tags at alle levels will be unique" + puts "" + puts "Sample 1: facet_samples.tcl 1 2" + puts "doc 1, path_ss=t1, path_ss=t2" + puts "" + puts "Sample 2: facet_samples.tcl 2 false 2 3" + puts "doc 1, path_ss=p1/t1, path_ss=p1/t2, path_ss=p1/t3," + puts " path_ss=p2/t1, path_ss=p2/t2, path_ss=p2/t3" + puts "doc 2, path_ss=p1/t1, path_ss=p1/t2, path_ss=p1/t3," + puts " path_ss=p2/t1, path_ss=p2/t2, path_ss=p2/t3" + puts "" + puts "Sample 3: facet_samples.tcl 3 true 2" + puts "doc 1, path_ss=t1_d1, path_ss=t2_d1" + puts "doc 2, path_ss=t1_d2, path_ss=t2_d2" + puts "doc 3, path_ss=t1_d3, path_ss=t2_d3" + puts "" + exit +} + +proc incgetcount { level } { + global uniques + global counters + + set u [lindex $counters $level] + set result $u + incr u + if { $u > [lindex $uniques $level] } { + set u 1 + } + lset counters $level $u + return $result +} + +# 2 2 -u 2 3 -> L0_T1/L1_T1, L0_T1/L1_T2, L0_T2/L1_T3, L0_T2/L1_T1 +proc getexp { level path } { + global elements + + set result {} + if { $level == [llength $elements] } { + return $path + } + if { $level > 0 } { + set path "$path/" + } + + for {set e 0} {$e < [lindex $elements $level]} {incr e} { + set val "L$level\_T[incgetcount $level]" + set result [concat $result [getexp [expr $level + 1] "$path$val"]] + } + return $result +} + +# 2 2 -u 2 3 -> L0_T1, L0_T1, L1_T1, L1_T2, L1_T3, L1_T1, L1_T2, L1_T3 +proc get792 { } { + global elements + + set result {} + set combos 1 + for {set level 0} {$level < [llength $elements]} {incr level} { + set c [lindex $elements $level] + set combos [expr $combos * $c] + + set path "L$level\_T" + for {set tag 0} {$tag < $combos} {incr tag} { + lappend result "$path[incgetcount $level]" + } + } + return $result +} + +proc print { list } { + foreach element $list { + puts -nonewline ",$element" + } +} + +proc makecounters { } { + global elements + + set counters {} + for {set c 0} {$c < [llength $elements]} {incr c} { + lappend counters 1 + } + return $counters +} + + +# Parse arguments +if { [llength $argv] < 2 } { + puts stderr "Too few arguments\n" + usage +} +if { [lindex $argv 0] == "-64" } { + set solr64 true + set argv [lrange $argv 1 end] +} else { + set solr64 false +} +set docs [lindex $argv 0] +set u [lsearch $argv "-u"] +if { $u > -1 } { + set uniques [lrange $argv [expr $u + 1] end] + set elements [lrange $argv 1 [expr $u - 1]] + if { !([llength $elements] == [llength $uniques]) } { + puts stderr "#elements must be equal to #uniques." + puts stderr "Got [llength $elements] elements ($elements) and [llength $uniques] uniques ($uniques) in arguments '$argv'" + usage + } +} else { + set elements [lrange $argv 1 end] + set uniques {} + for {set u 0} {$u < [llength $elements]} {incr u} { + lappend uniques 2147483648 + } +} +if { $solr64 } { + foreach element $elements { + if { !($element == 1) } { + puts stderr "When '-64' is specified, all elements must be 1. Got elements $elements" + usage + } + } +} + + +# Write header +set paths 1 +foreach element $elements { + set paths [expr $paths * $element] +} +puts -nonewline "id" +if { $solr64 } { + puts -nonewline ",levels_h" +} +for {set i 0} {$i < $paths} {incr i} { + puts -nonewline ",path_ss" +} +set combos 1 +for {set l 0} {$l < [llength $elements]} {incr l} { + set combos [expr $combos * [lindex $elements $l]] + for {set m 0} {$m < $combos} {incr m} { + if { [lindex $elements $l] == 1 } { + puts -nonewline ",level$l\_s" + } else { + puts -nonewline ",level$l\_ss" + } + } +} +puts "" + + +# Write paths + +set countersexp [makecounters] +set counters792 [makecounters] +for {set doc 1} {$doc <= $docs} {incr doc} { + puts -nonewline $doc +# set values [calculate_values $elements $uniques $counters] +# set counters [makepath "" $solr64 $doc 0 $elements $uniques $counters] + set counters $countersexp + set exp [getexp 0 {}] + set countersexp $counters + if { $solr64 } { + print $exp + } + print $exp + + set counters $counters792 + print [get792] + set counters792 $counters + puts "" +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/IdentityReader.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/IdentityReader.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/util/packed/IdentityReader.java (revision ) @@ -0,0 +1,28 @@ +package org.apache.lucene.util.packed; + +/** + * Returns the index as the value in {@link #get}. Useful for representing a 1:1 + * mapping of another structure. + */ +public class IdentityReader implements PackedInts.Reader { + private final int size; + + public IdentityReader(int size) { + this.size = size; + } + + @Override + public long get(int index) { + return index; + } + + @Override + public int getBitsPerValue() { + return 0; + } + + @Override + public int size() { + return size; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedIOFactory.java =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedIOFactory.java (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/exposed/ExposedIOFactory.java (revision ) @@ -0,0 +1,83 @@ +/* $Id:$ + * + * The Summa project. + * Copyright (C) 2005-2010 The State and University Library + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.apache.lucene.search.exposed; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +/** + * We need to use fixed Gap to get support for ordinals so we provide a factory + * for IndexReaders and IndexWriters for convenience. + */ +public class ExposedIOFactory { + public static boolean forceFixedCodec = false; + + public static IndexReader getReader(File location) throws IOException { + if (forceFixedCodec) { + return IndexReader.open( + FSDirectory.open(location), null, true, 1, getCodecProvider()); + } + return IndexReader.open(FSDirectory.open(location), true); + } + + public static IndexWriter getWriter(File location) throws IOException { + CodecProvider codecProvider = null; + if (forceFixedCodec) { + getCodecProvider(); + } + + Directory dir = FSDirectory.open(location); + IndexWriter writer = new IndexWriter( + dir, new IndexWriterConfig(Version.LUCENE_40, + new MockAnalyzer(new Random()))); + + writer.getConfig().setRAMBufferSizeMB(16.0); + + if (forceFixedCodec) { + writer.getConfig().setCodecProvider(codecProvider); + } + return writer; + } + + private static CodecProvider provider = null; + private static CodecProvider getCodecProvider() { + if (provider == null) { + provider = new CodecProvider(); +// codecProvider.register(new StandardCodec()); + FixedGapCodec fixedGapCodec = new FixedGapCodec(); + provider.register(fixedGapCodec); + provider.setDefaultFieldCodec(fixedGapCodec.name); + CodecProvider.setDefault(provider); + provider = provider; + } + return provider; + } +} Index: lucene/contrib/exposed/src/java/org/apache/lucene/search/package.html =================================================================== --- lucene/contrib/exposed/src/java/org/apache/lucene/search/package.html (revision ) +++ lucene/contrib/exposed/src/java/org/apache/lucene/search/package.html (revision ) @@ -0,0 +1,389 @@ + + + + + + + +Code to search indices. + +

Table Of Contents

+

+

    +
  1. Search Basics
    2. +
  2. The Query Classes
    3. +
  3. Changing the Scoring
    4. +
+

+ +

Search

+

+Search over indices. + +Applications usually call {@link +org.apache.lucene.search.Searcher#search(Query,int)} or {@link +org.apache.lucene.search.Searcher#search(Query,Filter,int)}. + + +

+ +

Query Classes

+

+ TermQuery +

+ +

Of the various implementations of + Query, the + TermQuery + is the easiest to understand and the most often used in applications. A TermQuery matches all the documents that contain the + specified + Term, + which is a word that occurs in a certain + Field. + Thus, a TermQuery identifies and scores all + Documents that have a Field with the specified string in it. + Constructing a TermQuery + is as simple as: + 

+        TermQuery tq = new TermQuery(new Term("fieldName", "term"));
+
In this example, the Query identifies all Documents that have the Field named "fieldName" + containing the word "term". +

+

+ BooleanQuery +

+ +

Things start to get interesting when one combines multiple + TermQuery instances into a BooleanQuery. + A BooleanQuery contains multiple + BooleanClauses, + where each clause contains a sub-query (Query + instance) and an operator (from BooleanClause.Occur) + describing how that sub-query is combined with the other clauses: +

    + +

  1. SHOULD — Use this operator when a clause can occur in the result set, but is not required. + If a query is made up of all SHOULD clauses, then every document in the result + set matches at least one of these clauses.

    2. + +

  2. MUST — Use this operator when a clause is required to occur in the result set. Every + document in the result set will match + all such clauses.

    3. + +

  3. MUST NOT — Use this operator when a + clause must not occur in the result set. No + document in the result set will match + any such clauses.

    4. +
+ Boolean queries are constructed by adding two or more + BooleanClause + instances. If too many clauses are added, a TooManyClauses + exception will be thrown during searching. This most often occurs + when a Query + is rewritten into a BooleanQuery with many + TermQuery clauses, + for example by WildcardQuery. + The default setting for the maximum number + of clauses 1024, but this can be changed via the + static method setMaxClauseCount + in BooleanQuery. +

+ +

Phrases

+ +

Another common search is to find documents containing certain phrases. This + is handled two different ways: +

    +
  1. +

    PhraseQuery + — Matches a sequence of + Terms. + PhraseQuery uses a slop factor to determine + how many positions may occur between any two terms in the phrase and still be considered a match.

    +
    2. +
  2. +

    SpanNearQuery + — Matches a sequence of other + SpanQuery + instances. SpanNearQuery allows for + much more + complicated phrase queries since it is constructed from other SpanQuery + instances, instead of only TermQuery + instances.

    +
    3. +
+

+ +

+ TermRangeQuery +

+ +

The + TermRangeQuery + matches all documents that occur in the + exclusive range of a lower + Term + and an upper + Term. + according to {@link java.lang.String#compareTo(String)}. It is not intended + for numerical ranges, use NumericRangeQuery instead. + + For example, one could find all documents + that have terms beginning with the letters a through c. This type of Query is frequently used to + find + documents that occur in a specific date range. +

+ +

+ NumericRangeQuery +

+ +

The + NumericRangeQuery + matches all documents that occur in a numeric range. + For NumericRangeQuery to work, you must index the values + using a special + NumericField. +

+ +

+ PrefixQuery, + WildcardQuery +

+ +

While the + PrefixQuery + has a different implementation, it is essentially a special case of the + WildcardQuery. + The PrefixQuery allows an application + to identify all documents with terms that begin with a certain string. The WildcardQuery generalizes this by allowing + for the use of * (matches 0 or more characters) and ? (matches exactly one character) wildcards. + Note that the WildcardQuery can be quite slow. Also + note that + WildcardQuery should + not start with * and ?, as these are extremely slow. + To remove this protection and allow a wildcard at the beginning of a term, see method + setAllowLeadingWildcard in + QueryParser. +

+

+ FuzzyQuery +

+ +

A + FuzzyQuery + matches documents that contain terms similar to the specified term. Similarity is + determined using + Levenshtein (edit) distance. + This type of query can be useful when accounting for spelling variations in the collection. +

+ +

Changing Similarity

+ +

Chances are DefaultSimilarity is sufficient for all + your searching needs. + However, in some applications it may be necessary to customize your Similarity implementation. For instance, some + applications do not need to + distinguish between shorter and longer documents (see a "fair" similarity).

+ +

To change Similarity, one must do so for both indexing and + searching, and the changes must happen before + either of these actions take place. Although in theory there is nothing stopping you from changing mid-stream, it + just isn't well-defined what is going to happen. +

+ +

To make this change, implement your own Similarity (likely + you'll want to simply subclass + DefaultSimilarity) and then use the new + class by calling + IndexWriter.setSimilarity + before indexing and + Searcher.setSimilarity + before searching. +

+ +

+ If you are interested in use cases for changing your similarity, see the Lucene users's mailing list at Overriding Similarity. + In summary, here are a few use cases: +

    +

  1. SweetSpotSimilarity — SweetSpotSimilarity gives small increases + as the frequency increases a small amount + and then greater increases when you hit the "sweet spot", i.e. where you think the frequency of terms is + more significant.

    2. +

  2. Overriding tf — In some applications, it doesn't matter what the score of a document is as long as a + matching term occurs. In these + cases people have overridden Similarity to return 1 from the tf() method.

    3. +

  3. Changing Length Normalization — By overriding lengthNorm, + it is possible to discount how the length of a field contributes + to a score. In DefaultSimilarity, + lengthNorm = 1 / (numTerms in field)^0.5, but if one changes this to be + 1 / (numTerms in field), all fields will be treated + "fairly".

    4. +
+ In general, Chris Hostetter sums it up best in saying (from the Lucene users's mailing list): +
[One would override the Similarity in] ... any situation where you know more about your data then just + that + it's "text" is a situation where it *might* make sense to to override your + Similarity method.
+

+ +

Changing Scoring — Expert Level

+ +

Changing scoring is an expert level task, so tread carefully and be prepared to share your code if + you want help. +

+ +

With the warning out of the way, it is possible to change a lot more than just the Similarity + when it comes to scoring in Lucene. Lucene's scoring is a complex mechanism that is grounded by + three main classes: +

    +
  1. + Query — The abstract object representation of the + user's information need.
    2. +
  2. + Weight — The internal interface representation of + the user's Query, so that Query objects may be reused.
    3. +
  3. + Scorer — An abstract class containing common + functionality for scoring. Provides both scoring and explanation capabilities.
    4. +
+ Details on each of these classes, and their children, can be found in the subsections below. +

+

The Query Class

+

In some sense, the + Query + class is where it all begins. Without a Query, there would be + nothing to score. Furthermore, the Query class is the catalyst for the other scoring classes as it + is often responsible + for creating them or coordinating the functionality between them. The + Query class has several methods that are important for + derived classes: +

    +
  1. createWeight(Searcher searcher) — A + Weight is the internal representation of the + Query, so each Query implementation must + provide an implementation of Weight. See the subsection on The Weight Interface below for details on implementing the Weight + interface.
    2. +
  2. rewrite(IndexReader reader) — Rewrites queries into primitive queries. Primitive queries are: + TermQuery, + BooleanQuery, and other queries that implement Query.html#createWeight(Searcher searcher)
    3. +
+

+

The Weight Interface

+

The + Weight + interface provides an internal representation of the Query so that it can be reused. Any + Searcher + dependent state should be stored in the Weight implementation, + not in the Query class. The interface defines six methods that must be implemented: +

    +
  1. + Weight#getQuery() — Pointer to the + Query that this Weight represents.
    2. +
  2. + Weight#getValue() — The weight for + this Query. For example, the TermQuery.TermWeight value is + equal to the idf^2 * boost * queryNorm
    3. +
  3. + + Weight#sumOfSquaredWeights() — The sum of squared weights. For TermQuery, this is (idf * + boost)^2
    4. +
  4. + + Weight#normalize(float) — Determine the query normalization factor. The query normalization may + allow for comparing scores between queries.
    5. +
  5. + + Weight#scorer(IndexReader, boolean, boolean) — Construct a new + Scorer + for this Weight. See + The Scorer Class + below for help defining a Scorer. As the name implies, the + Scorer is responsible for doing the actual scoring of documents given the Query. +
    6. +
  6. + + Weight#explain(Searcher, IndexReader, int) — Provide a means for explaining why a given document was + scored + the way it was.
    7. +
+

+

The Scorer Class

+

The + Scorer + abstract class provides common scoring functionality for all Scorer implementations and + is the heart of the Lucene scoring process. The Scorer defines the following abstract (some of them are not + yet abstract, but will be in future versions and should be considered as such now) methods which + must be implemented (some of them inherited from DocIdSetIterator ): +

    +
  1. + DocIdSetIterator#nextDoc() — Advances to the next + document that matches this Query, returning true if and only + if there is another document that matches.
    2. +
  2. + DocIdSetIterator#docID() — Returns the id of the + Document + that contains the match. It is not valid until next() has been called at least once. +
    3. +
  3. + Scorer#score(Collector) — + Scores and collects all matching documents using the given Collector. +
    4. +
  4. + Scorer#score() — Return the score of the + current document. This value can be determined in any + appropriate way for an application. For instance, the + TermScorer + returns the tf * Weight.getValue() * fieldNorm. +
    5. +
  5. + DocIdSetIterator#advance(int) — Skip ahead in + the document matches to the document whose id is greater than + or equal to the passed in value. In many instances, advance can be + implemented more efficiently than simply looping through all the matching documents until + the target document is identified.
    6. +
+

+

Why would I want to add my own Query?

+ +

In a nutshell, you want to add your own custom Query implementation when you think that Lucene's + aren't appropriate for the + task that you want to do. You might be doing some cutting edge research or you need more information + back + out of Lucene (similar to Doug adding SpanQuery functionality).

+ + +