eldest) {
+ return size() > cacheSizeNum;
+ }
+ };
+ }
+
+ protected abstract T lookup(final long index) throws IOException;
+
+ public T get(final long index) throws IOException {
+ requests++;
+ T entry = cache.get(index);
+ if (entry != null) {
+ return entry;
+ }
+ misses++;
+
+ long startLookup = System.nanoTime();
+ entry = lookup(index);
+ lookupTime += System.nanoTime() - startLookup;
+ lookups++;
+ cache.put(index, entry);
+ readAhead(index+1, readAheadNum);
+ return entry;
+
+ }
+
+ private void readAhead(final long startIndex, final int readAhead)
+ throws IOException {
+ long index = startIndex;
+ int readsLeft = readAhead;
+ while (true) {
+ if (readsLeft == 0 || index > finalIndex||
+ (onlyReadAheadIfSpace && cache.size() >= cacheSizeNum)) {
+ break;
+ }
+ readAheadRequests++;
+ T entry = cache.get(index);
+ if (entry == null) {
+ long startLookup = System.nanoTime();
+ entry = lookup(index);
+ lookupTime += System.nanoTime() - startLookup;
+ lookups++;
+ if (entry == null) {
+ break;
+ }
+/* if (index > 19999) {
+ System.out.println("Putting " + index);
+ }*/
+ cache.put(index, entry);
+ } else if (stopReadAheadOnExistingValue) {
+ break;
+ }
+ index++;
+ readsLeft--;
+ }
+ }
+
+ /**
+ * Removes the cache entry for the given index from cache is present.
+ *
+ * If the user of the cache knows that the String for a given ordinal should
+ * not be used again, calling this method helps the cache to perform better.
+ * @param index the ordinal for the elementto remove.
+ * @return the old element if it was present in the cache.
+ */
+ public T release(final long index) {
+ return cache.remove(index);
+ }
+
+ /**
+ * Clears the cache.
+ */
+ public void clear() {
+ cache.clear();
+ requests = 0;
+ lookups = 0;
+ readAheadRequests = 0;
+ misses = 0;
+ lookupTime = 0;
+ }
+
+ public int getCacheSize() {
+ return cacheSizeNum;
+ }
+
+ /**
+ * Setting the cache size clears the cache.
+ * @param cacheSize the new size of the cache, measured in elements.
+ */
+ public void setCacheSize(int cacheSize) {
+ this.cacheSizeNum = cacheSize;
+ cache.clear();
+ }
+
+ public int getReadAhead() {
+ return readAheadNum;
+ }
+
+ public void setReadAhead(int readAhead) {
+ this.readAheadNum = readAhead;
+ }
+
+ public boolean isOnlyReadAheadIfSpace() {
+ return onlyReadAheadIfSpace;
+ }
+
+ public void setOnlyReadAheadIfSpace(boolean onlyReadAheadIfSpace) {
+ this.onlyReadAheadIfSpace = onlyReadAheadIfSpace;
+ }
+
+ public boolean isStopReadAheadOnExistingValue() {
+ return stopReadAheadOnExistingValue;
+ }
+
+ public void setStopReadAheadOnExistingValue(
+ boolean stopReadAheadOnExistingValue) {
+ this.stopReadAheadOnExistingValue = stopReadAheadOnExistingValue;
+ }
+
+ public String getStats() {
+ return "CachedProvider cacheSize=" + cacheSizeNum
+ + ", misses=" + misses + "/" + requests
+ + ", lookups=" + lookups + " (" + lookupTime / 1000000 + " ms ~= "
+ + (lookupTime == 0 ? "N/A" : lookups * 1000000 / lookupTime)
+ + " lookups/ms), readAheads=" + readAheadRequests;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,31 @@
+package org.apache.lucene.search.exposed;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+
+public class CachedCollatorKeyProvider extends CachedProvider {
+ private TermProvider source;
+ private final Collator collator;
+
+ /**
+ * @param source the backing term provider.
+ * @param collator the Collator used for key generation.
+ * @param cacheSize the maximum number of elements to hold in cache.
+ * @param readAhead the maximum number of lookups that can be performed
+ * after a plain lookup.
+ * @throws java.io.IOException if the cache could access the source.
+ */
+ public CachedCollatorKeyProvider(
+ TermProvider source, Collator collator,
+ int cacheSize, int readAhead) throws IOException {
+ super(cacheSize, readAhead, source.getOrdinalTermCount()-1);
+ this.source = source;
+ this.collator = collator;
+ }
+
+ @Override
+ protected CollationKey lookup(final long index) throws IOException {
+ return collator.getCollationKey(source.getTerm(index).utf8ToString());
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java Tue Sep 21 10:36:06 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java Tue Sep 21 10:36:06 CEST 2010
@@ -0,0 +1,216 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.*;
+
+public class ExposedCache implements FieldCache {
+ private final FieldCache fallback;
+
+ private final ArrayList cache = new ArrayList(5);
+ private final List remoteCaches =
+ new ArrayList();
+
+ private static final ExposedCache exposedCache;
+ static {
+ exposedCache = new ExposedCache(FieldCache.DEFAULT);
+ }
+ public static ExposedCache getInstance() {
+ return exposedCache;
+ }
+
+ public ExposedCache(FieldCache fallback) {
+ this.fallback = fallback;
+ }
+
+ public void addRemoteCache(PurgeCallback callback) {
+ remoteCaches.add(callback);
+ }
+
+ public boolean removeRemoteCache(PurgeCallback callback) {
+ return remoteCaches.remove(callback);
+ }
+
+ public TermProvider getProvider(
+ IndexReader reader, ExposedRequest.Group group) throws IOException {
+ return getProvider(reader, group.getName(), group.getFieldNames(),
+ group.getComparator(), group.getComparatorID());
+ }
+
+ public TermProvider getProvider(
+ IndexReader reader, String groupName, List fieldNames,
+ Comparator comparator, String comparatorID) throws IOException {
+ ExposedRequest.Group groupRequest = ExposedRequest.createGroup(
+ groupName, fieldNames, comparator, comparatorID);
+
+ for (TermProvider provider: cache) {
+ if (provider instanceof GroupTermProvider
+ && ((GroupTermProvider) provider).getRequest().worksfor(groupRequest)
+ && provider.getReaderHash() == reader.hashCode()) {
+ return provider;
+ }
+ }
+ System.out.println("Creating provider for " + groupName);
+
+ // No cached value. Modify the comparator IDs to LUCENE-order if they were
+ // stated as free. Aw we create the query ourselves, this is okay.
+ groupRequest.normalizeComparatorIDs();
+
+ IndexReader[] readers;
+ if (reader.getSequentialSubReaders() == null) {
+ readers = new IndexReader[1];
+ readers[0] = reader;
+ } else {
+ readers = reader.getSequentialSubReaders();
+ }
+
+ List fieldProviders =
+ new ArrayList(readers.length * fieldNames.size());
+
+ for (IndexReader sub: readers) {
+ for (ExposedRequest.Field fieldRequest: groupRequest.getFields()) {
+ fieldProviders.add(getProvider(sub, fieldRequest, true, true));
+ }
+ }
+ TermProvider groupProvider = new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ cache.add(groupProvider);
+ return groupProvider;
+ }
+
+ FieldTermProvider getProvider(
+ IndexReader segmentReader, ExposedRequest.Field request,
+ boolean cacheTables, boolean cacheProvider) throws IOException {
+ for (TermProvider provider: cache) {
+ if (provider instanceof FieldTermProvider) {
+ if (provider.getRecursiveHash() == segmentReader.hashCode()
+ && ((FieldTermProvider)provider).getRequest().equals(request)) {
+ return (FieldTermProvider)provider;
+ }
+ }
+ }
+ FieldTermProvider provider =
+ new FieldTermProvider(segmentReader, request, cacheTables);
+ if (cacheProvider) {
+ cache.add(provider);
+ }
+ return provider;
+ }
+
+ public void purgeAllCaches() {
+ fallback.purgeAllCaches();
+ cache.clear();
+ for (PurgeCallback purger: remoteCaches) {
+ purger.purgeAllCaches();
+ }
+ }
+
+ /**
+ * Purges all entries with connections to the given index reader. For exposed
+ * structures that means all structures except the ones that are
+ * FieldTermProviders not relying on the given index reader.
+ * @param r the reader to purge.
+ */
+ public synchronized void purge(IndexReader r) {
+ fallback.purge(r);
+ Iterator remover =
+ cache.iterator();
+ while (remover.hasNext()) {
+ TermProvider provider = remover.next();
+ if (!(provider instanceof FieldTermProvider) ||
+ provider.getRecursiveHash() == r.hashCode()) {
+ remover.remove();
+ }
+ }
+ for (PurgeCallback purger: remoteCaches) {
+ purger.purge(r);
+ }
+ }
+
+ public static interface PurgeCallback {
+ void purgeAllCaches();
+ void purge(IndexReader r);
+ }
+
+ /* Direct delegations to the default cache */
+
+ public byte[] getBytes(IndexReader reader, String field) throws IOException {
+ return fallback.getBytes(reader, field);
+ }
+
+ public byte[] getBytes(IndexReader reader, String field, ByteParser parser) throws IOException {
+ return fallback.getBytes(reader, field, parser);
+ }
+
+ public short[] getShorts(IndexReader reader, String field) throws IOException {
+ return fallback.getShorts(reader, field);
+ }
+
+ public short[] getShorts(IndexReader reader, String field, ShortParser parser) throws IOException {
+ return fallback.getShorts(reader, field, parser);
+ }
+
+ public int[] getInts(IndexReader reader, String field) throws IOException {
+ return fallback.getInts(reader, field);
+ }
+
+ public int[] getInts(IndexReader reader, String field, IntParser parser) throws IOException {
+ return fallback.getInts(reader, field, parser);
+ }
+
+ public float[] getFloats(IndexReader reader, String field) throws IOException {
+ return fallback.getFloats(reader, field);
+ }
+
+ public float[] getFloats(IndexReader reader, String field, FloatParser parser) throws IOException {
+ return fallback.getFloats(reader, field, parser);
+ }
+
+ public long[] getLongs(IndexReader reader, String field) throws IOException {
+ return fallback.getLongs(reader, field);
+ }
+
+ public long[] getLongs(IndexReader reader, String field, LongParser parser) throws IOException {
+ return fallback.getLongs(reader, field, parser);
+ }
+
+ public double[] getDoubles(IndexReader reader, String field) throws IOException {
+ return fallback.getDoubles(reader, field);
+ }
+
+ public double[] getDoubles(IndexReader reader, String field, DoubleParser parser) throws IOException {
+ return fallback.getDoubles(reader, field, parser);
+ }
+
+ public DocTerms getTerms(IndexReader reader, String field) throws IOException {
+ return fallback.getTerms(reader, field);
+ }
+
+ public DocTerms getTerms(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return fallback.getTerms(reader, field, fasterButMoreRAM);
+ }
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field) throws IOException {
+ return fallback.getTermsIndex(reader, field);
+ }
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return fallback.getTermsIndex(reader, field, fasterButMoreRAM);
+ }
+
+ public FieldCache.CacheEntry[] getCacheEntries() {
+ return fallback.getCacheEntries();
+ }
+
+ public void setInfoStream(PrintStream stream) {
+ fallback.setInfoStream(stream);
+ }
+
+ public PrintStream getInfoStream() {
+ return fallback.getInfoStream();
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,138 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.GrowingMutable;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+
+/**
+ * Default implementation of some methods from TermProvider.
+ */
+public abstract class TermProviderImpl implements TermProvider {
+ protected boolean cacheTables;
+ private PackedInts.Mutable docToSingle = null;
+
+ private final Comparator comparator;
+ private final String comparatorID;
+ private final String designation;
+
+ /**
+ * @param comparator the comparator used for sorting. If this is null, the
+ * natural BytesRef-order is used.
+ * @param comparatorID an ID for the comparator. If this is natural order,
+ * this should be {@link ExposedRequest#LUCENE_ORDER}.
+ * @param designation used for feedback and debugging. No formal requirements.
+ * @param cacheTables if true, tables such as orderedOrdinals, docID2indirect
+ * and similar should be cached for re-requests after generation.
+ */
+ protected TermProviderImpl(
+ Comparator comparator, String comparatorID,
+ String designation, boolean cacheTables) {
+ this.comparator = comparator;
+ this.comparatorID = comparatorID;
+ this.cacheTables = cacheTables;
+ this.designation = designation;
+ }
+
+ public synchronized PackedInts.Reader getDocToSingleIndirect()
+ throws IOException {
+ if (docToSingle != null) {
+ return docToSingle;
+ }
+ if (getMaxDoc() > Integer.MAX_VALUE) {
+ throw new UnsupportedOperationException(
+ "Unable to handle more than Integer.MAX_VALUE documents. " +
+ "Got macDocs " + getMaxDoc());
+ }
+ // TODO: getMaxDoc() is seriously wonky for GroupTermProvider. What breaks?
+ long sortTime;
+ // TODO: Check why it is extremely slow to start with low maxValue
+ docToSingle = new GrowingMutable(
+ 0, (int)getMaxDoc(), -1, getOrdinalTermCount(),
+ ExposedSettings.priority == ExposedSettings.PRIORITY.speed);
+ try {
+ Iterator ei = getIterator(true);
+ sortTime = System.currentTimeMillis();
+ while (ei.hasNext()) {
+ ExposedTuple tuple = ei.next();
+ docToSingle.set((int)tuple.docID, tuple.indirect);
+/* if (tuple.indirect << 60 == 0) {
+ System.out.print(".");
+ }*/
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to create doc to indirect map", e);
+ }
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println(this.getClass().getSimpleName() + " merge-mapped "
+ + (getMaxDoc()-1) + " docs to single terms in " + sortTime + " ms: " +
+ (sortTime == 0 ? "N/A" : docToSingle.size() / sortTime) + " docs/ms");
+ return docToSingle;
+ }
+
+ public String getDesignation() {
+ return designation;
+ }
+
+ public Comparator getComparator() {
+ return comparator;
+ }
+
+ public String getComparatorID() {
+ return comparatorID;
+ }
+
+ @Override
+ public String toString() {
+ return "TermProvider(" + getDesignation() + ")";
+ }
+
+ public int getNearestTermIndirect(BytesRef key) throws IOException {
+ return getNearestTermIndirect(key, 0, (int)getUniqueTermCount());
+ }
+ public int getNearestTermIndirect(
+ final BytesRef key, int startTermPos, int endTermPos) throws IOException {
+ if (getComparator() != null) {
+ return getNearestWithComparator(key, startTermPos, endTermPos);
+ }
+ int low = startTermPos;
+ int high = endTermPos-1;
+ while (low <= high) {
+ int middle = low + high >>> 1;
+ BytesRef midVal = getOrderedTerm(middle);
+ int compValue = midVal.compareTo(key);
+ if (compValue < 0) {
+ low = middle + 1;
+ } else if (compValue > 0) {
+ high = middle - 1;
+ } else {
+ return middle;
+ }
+ }
+ return low;
+ }
+
+ private int getNearestWithComparator(
+ final BytesRef key, int startTermPos, int endTermPos) throws IOException {
+ final Comparator comparator = getComparator();
+ int low = startTermPos;
+ int high = endTermPos-1;
+ while (low <= high) {
+ int middle = low + high >>> 1;
+ BytesRef midVal = getOrderedTerm(middle);
+ int compValue = comparator.compare(midVal, key);
+ if (compValue < 0) {
+ low = middle + 1;
+ } else if (compValue > 0) {
+ high = middle - 1;
+ } else {
+ return middle;
+ }
+ }
+ return low;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java Thu Sep 23 11:42:36 CEST 2010
+++ lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java Thu Sep 23 11:42:36 CEST 2010
@@ -0,0 +1,210 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A PackedInts that grows to accommodate any legal data that is added.
+ * Deltas for both array length and values are used so memory-representation
+ * is fairly compact.
+ *
+ * The initial default value for a growing mutable is the given minValue or 0
+ * if no minValue is specified. If a value smaller than minValue is added after
+ * construction, the new default value will be that value and the old non-set
+ * values will still be the old default value. To put it in code: {@code
+ GrowingMutable g = new GrowingMutable(0, 5, 5, 10);
+ assert(g.get(0) == 5);
+ g.set(2, 4);
+ assert(g.get(0) == 5);
+ assert(g.get(1) == 4);
+ }. To avoid error it is imperative to either specify a minValue that is
+ guaranteed to be the minimum value for the life of the mutable or to ensure
+ that all values are assigned explicitly.
+ *
+ * The penalty for using this implementation compared to a plain PackedInts is
+ * slower {@link #set}s as it needs to check boundaries and potentially expand
+ * the internal array. There are no conditionals and very little extra overhead
+ * for {@link #get}s so access is still fast.
+ */
+// TODO: Optimize the speed of array length growth
+public class GrowingMutable implements PackedInts.Mutable {
+ // Grow array length with this factor beyond the immediate need
+ private static final double GROWTH_FACTOR = 1.2;
+
+ private int indexMin = 0;
+ private int indexMax = 0;
+ private long valueMin = 0;
+ private long valueMax = 0;
+ private int size = 0; // Size is explicit to handle grow seamlessly
+ private boolean optimizeSpeed = false;
+
+ private PackedInts.Mutable values;
+
+ /**
+ * Create an empty array of size 0. If it is possible to estimate the amount
+ * and rages of the values to add to the array, it is recommended to use
+ * {@link #GrowingMutable(int,int,long,long,boolean)} instead.
+ */
+ public GrowingMutable() {
+ values = PackedInts.getMutable(0, 1);
+ }
+
+ /**
+ * Create an empty array with the given boundaries. The boundaries will be
+ * expanded as needed. The default value is valueMin.
+ * @param indexMin the minimum index for the array.
+ * @param indexMax the maximum index for the array.
+ * @param valueMin the minimum value for the array.
+* Note that this can be negative.
+ * @param valueMax the maximum value for the array. This must be higher than
+* or equal to valueMin.
+ * @param optimizeSpeed if true, the number of bit required to represent the
+ * values are rounded to 1, 2, 3, 4, 8, 16, 32-40 or 64.
+ */
+ public GrowingMutable(int indexMin, int indexMax,
+ long valueMin, long valueMax, boolean optimizeSpeed) {
+ if (indexMin > indexMax) {
+ throw new IllegalArgumentException(String.format(
+ "indexMin(%d) must be <= indexMax(%d)", indexMin, indexMax));
+ }
+ if (valueMin > valueMax) {
+ throw new IllegalArgumentException(String.format(
+ "valueMin(%d) must be <= valueMax(%d)", valueMin, valueMax));
+ }
+ this.indexMin = indexMin;
+ this.indexMax = indexMax;
+ this.valueMin = valueMin;
+ this.valueMax = valueMax;
+ this.size = indexMax - indexMin + 1;
+ this.optimizeSpeed = optimizeSpeed;
+ values = getMutable(
+ indexMax - indexMin + 1, PackedInts.bitsRequired(valueMax - valueMin));
+ }
+
+ private PackedInts.Mutable getMutable(int valueCount, int bitsPerValue) {
+ if (!optimizeSpeed) {
+ return PackedInts.getMutable(valueCount, bitsPerValue);
+ }
+ if (bitsPerValue <= 4 ||
+ (bitsPerValue > 32 && bitsPerValue < 40)) { // TODO: Consider the 40
+ return PackedInts.getMutable(valueCount, bitsPerValue);
+ }
+ return PackedInts.getMutable(valueCount, bitsPerValue);
+ }
+
+ // Make sure that there is room. Note: This might adjust the boundaries.
+ private synchronized void checkValues(final int index, final long value) {
+ if (index >= indexMin && index <= indexMax
+ && value >= valueMin && value <= valueMax) {
+ return;
+ }
+ // TODO: Just adjust max is within bitsPerValue
+ if (size() == 0) { // First addition
+ indexMin = index;
+ indexMax = index;
+ valueMin = value;
+ valueMax = value;
+ GrowingMutable newGrowingMutable = new GrowingMutable(
+ indexMin, indexMax, valueMin, valueMax, optimizeSpeed);
+ size = 1;
+ values = newGrowingMutable.values;
+ return;
+ }
+
+ // indexes are grown with factor, values without
+ int newIndexMin =
+ index >= indexMin ? indexMin :
+ indexMax - (int)Math.ceil((indexMax - index) * GROWTH_FACTOR);
+ int newIndexMax =
+ index <= indexMax ? indexMax :
+ indexMin + (int)Math.ceil((index - indexMin) * GROWTH_FACTOR);
+ long newValueMin = Math.min(value, valueMin);
+ long newValueMax = Math.max(value, valueMax);
+
+ GrowingMutable newGrowingMutable = new GrowingMutable(
+ newIndexMin, newIndexMax, newValueMin, newValueMax, optimizeSpeed);
+ for (int i = indexMin ; i <= indexMax ; i++) {
+ newGrowingMutable.set(i, get(i));
+ }
+
+ values = newGrowingMutable.values;
+ indexMin = newIndexMin;
+ indexMax = newIndexMax;
+ valueMin = newValueMin;
+ valueMax = newValueMax;
+ size = indexMax - indexMin; // Size is always minimum to accommodate index
+ }
+
+ public void set(int index, long value) {
+ checkValues(index, value);
+ values.set(index - indexMin, value - valueMin);
+ }
+
+ @Override
+ public String toString() {
+ final int DUMP_SIZE = 10;
+ StringBuffer sb = new StringBuffer(DUMP_SIZE * 10);
+ sb.append("GrowingMutable(");
+ for (int i = indexMin ; i < indexMin + Math.min(size(), DUMP_SIZE) ; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append("i(").append(Integer.toString(i)).append(")=");
+ sb.append(Long.toString(get(i)));
+ }
+ if (size > DUMP_SIZE) {
+ sb.append(", ...");
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ /**
+ * Clear only resets the array. It does not free memory or adjust size.
+ */
+ public void clear() {
+ values.clear();
+ }
+
+ public long get(int index) {
+ return values.get(index - indexMin) + valueMin;
+ }
+
+ public int getBitsPerValue() {
+ return values.getBitsPerValue();
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public int getIndexMin() {
+ return indexMin;
+ }
+
+ public int getIndexMax() {
+ return indexMax;
+ }
+
+ public long getValueMin() {
+ return valueMin;
+ }
+
+ public long getValueMax() {
+ return valueMax;
+ }
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/facet/performance.txt
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/facet/performance.txt Thu Sep 23 10:09:45 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/facet/performance.txt Thu Sep 23 10:09:45 CEST 2010
@@ -0,0 +1,229 @@
+Some results from TestExposedFacets.testScaleFacet on a Dell M6500 laptop:
+i7 processor @1.7GHz, PC1333 RAM, SSD.
+
+********************************************************************************
+
+*************** Finished testing in 2:35 minutes. Result follows
+Speed:
+
+Index = /home/te/projects/index2M (2000000 documents)
+used heap after loading index and performing a simple search: 9 MB
+Maximum possible memory (Runtime.getRuntime().maxMemory()): 1761 MB
+
+First natural order sorted search for "even:true" with 1000000 hits: 0:03 minutes
+Subsequent 5 sorted searches average response time: 43 ms
+Hit #0 was doc #0 with id 00000000
+Hit #1 was doc #2 with id 00000002
+Hit #2 was doc #4 with id 00000004
+Hit #3 was doc #6 with id 00000006
+Hit #4 was doc #8 with id 00000008
+
+Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 1:52 minutes
+First faceting for even:true: 249 ms
+Subsequent 4 faceting calls (count caching disabled) response times: 161 ms
+
+
+
+ a_ 000J6iWC743074
+ a_00 0nWARnepn NC0zeR509161
+ a_000O påøFÅoPN1000146
+ a_000UO0KEOzGkXbW1957082
+ a_000wkNk4f wx9 åd879060
+
+
+ a_T1678792
+ a_E1319214
+ a_000UO0KEOzGkXbW1957082
+ a_001017844
+ a_å ååØLcmoJNæ7KrBzX320280
+
+
+ S
+ A
+ W
+ G
+ L
+
+
+
+Initial lookup pool request (might result in structure building): 0:37 minutes
+First index lookup for "even:true": 66 ms
+Subsequent 91 index lookups average response times: 0 ms
+
+
+
+ a_xsQb0KkodE68366884
+ a_xSQBnGFÅLÆ6cZuæNsb696651
+ a_xsqBØ1n r7 ÆL355064
+ a_Xsqco DP møKshZIyUg1575306
+ a_XSqeo Kck 3g720748
+
+
+
+First natural order sorted search for "multi:A" with 190198 hits: 11 ms
+Subsequent 5 sorted searches average response time: 8 ms
+Hit #0 was doc #8 with id 00000008
+Hit #1 was doc #36 with id 00000036
+Hit #2 was doc #48 with id 00000048
+Hit #3 was doc #63 with id 00000063
+Hit #4 was doc #69 with id 00000069
+
+Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms
+First faceting for multi:A: 44 ms
+Subsequent 4 faceting calls (count caching disabled) response times: 44 ms
+
+
+
+ a_ 000J6iWC743074
+ a_00 0nWARnepn NC0zeR509161
+ a_000O påøFÅoPN1000146
+ a_000UO0KEOzGkXbW1957082
+ a_000wkNk4f wx9 åd879060
+
+
+ a_001019122
+ a_001376513
+ a_001532258
+ a_001948685
+ a_ÅåÅpp1909514
+
+
+ A
+ E
+ V
+ D
+ J
+
+
+
+Initial lookup pool request (might result in structure building): 0 ms
+First index lookup for "multi:A": 19 ms
+Subsequent 91 index lookups average response times: 0 ms
+
+
+
+ a_xsQb0KkodE68366884
+ a_xSQBnGFÅLÆ6cZuæNsb696651
+ a_xsqBØ1n r7 ÆL355064
+ a_Xsqco DP møKshZIyUg1575306
+ a_XSqeo Kck 3g720748
+
+
+
+
+Free memory with sort, facet and index lookup structures intact: 145 MB
+
+********************************************************************************
+Memory:
+*************** Finished testing in 2:34 minutes. Result follows
+
+Index = /home/te/projects/index2M (2000000 documents)
+used heap after loading index and performing a simple search: 9 MB
+Maximum possible memory (Runtime.getRuntime().maxMemory()): 1761 MB
+
+First natural order sorted search for "even:true" with 1000000 hits: 0:03 minutes
+Subsequent 5 sorted searches average response time: 46 ms
+Hit #0 was doc #0 with id 00000000
+Hit #1 was doc #2 with id 00000002
+Hit #2 was doc #4 with id 00000004
+Hit #3 was doc #6 with id 00000006
+Hit #4 was doc #8 with id 00000008
+
+Facet pool acquisition for for "even:true" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 1:49 minutes
+First faceting for even:true: 288 ms
+Subsequent 4 faceting calls (count caching disabled) response times: 242 ms
+
+
+
+ a_ 000J6iWC743074
+ a_00 0nWARnepn NC0zeR509161
+ a_000O påøFÅoPN1000146
+ a_000UO0KEOzGkXbW1957082
+ a_000wkNk4f wx9 åd879060
+
+
+ a_T1678792
+ a_E1319214
+ a_000UO0KEOzGkXbW1957082
+ a_001017844
+ a_å ååØLcmoJNæ7KrBzX320280
+
+
+ S
+ A
+ W
+ G
+ L
+
+
+
+Initial lookup pool request (might result in structure building): 0:39 minutes
+First index lookup for "even:true": 115 ms
+Subsequent 91 index lookups average response times: 0 ms
+
+
+
+ a_xsQb0KkodE68366884
+ a_xSQBnGFÅLÆ6cZuæNsb696651
+ a_xsqBØ1n r7 ÆL355064
+ a_Xsqco DP møKshZIyUg1575306
+ a_XSqeo Kck 3g720748
+
+
+
+First natural order sorted search for "multi:A" with 190198 hits: 11 ms
+Subsequent 5 sorted searches average response time: 11 ms
+Hit #0 was doc #8 with id 00000008
+Hit #1 was doc #36 with id 00000036
+Hit #2 was doc #48 with id 00000048
+Hit #3 was doc #63 with id 00000063
+Hit #4 was doc #69 with id 00000069
+
+Facet pool acquisition for for "multi:A" with structure groups(group(name=sorted, order=locale, locale=da, fields(a)), group(name=count, order=count, locale=null, fields(a)), group(name=multi, order=count, locale=null, fields(facet))): 0 ms
+First faceting for multi:A: 67 ms
+Subsequent 4 faceting calls (count caching disabled) response times: 73 ms
+
+
+
+ a_ 000J6iWC743074
+ a_00 0nWARnepn NC0zeR509161
+ a_000O påøFÅoPN1000146
+ a_000UO0KEOzGkXbW1957082
+ a_000wkNk4f wx9 åd879060
+
+
+ a_001019122
+ a_001376513
+ a_001532258
+ a_001948685
+ a_ÅåÅpp1909514
+
+
+ A
+ E
+ V
+ D
+ J
+
+
+
+Initial lookup pool request (might result in structure building): 0 ms
+First index lookup for "multi:A": 29 ms
+Subsequent 91 index lookups average response times: 0 ms
+
+
+
+ a_xsQb0KkodE68366884
+ a_xSQBnGFÅLÆ6cZuæNsb696651
+ a_xsqBØ1n r7 ÆL355064
+ a_Xsqco DP møKshZIyUg1575306
+ a_XSqeo Kck 3g720748
+
+
+
+
+Free memory with sort, facet and index lookup structures intact: 111 MB
+
+********************************************************************************
+********************************************************************************
+********************************************************************************
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java Sun Sep 19 20:11:30 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/ExposedFacets.java Sun Sep 19 20:11:30 CEST 2010
@@ -0,0 +1,83 @@
+package org.apache.lucene.search.exposed.facet;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+
+import java.io.IOException;
+
+/**
+ * ExposedFacets relies on the ordinal-oriented structures from the exposed
+ * framework. It shares structures with sorters whenever possible.
+ *
+ * The overall goal of exposed faceting is to minimize memory usage for
+ * faceting. This comes at a slight cost in processing speed as PackedInts
+ * are used to hold arrays of integers instead of int[]. The faceting system
+ * used delayed resolving of tags, meaning that only the returned tags are
+ * resolved from internal IDs to Strings. The resolve process taxes the
+ * IO-system and will likely be slow on a setup with a small disk cache,
+ * large index and spinning disks. For those not in-the-know, switching to
+ * SSDs for the index is the current silver bullet in the year 2010 for search
+ * responsiveness in general and works very well with the exposed framework.
+ *
+ * For each facet, a {@link org.apache.lucene.search.exposed.TermProvider}
+ * handles the ordinal to term mapping.
+ * This allows for locale based sorting of tags and similar. A supermap is
+ * created from document id to TermProvider entries by treating the TermProvider
+ * entry positions as logically increasing. Example: TermProvider A has 10
+ * entries, provider B has 20 entries. When a document is mapped to entry 5
+ * from provider B, the value 10 + 5 is stored in the map.
+ *
+ * The supermap is logically a two-dimensional integer array where
+ * supermap[docID] contains an array of TermProvider entry positions.
+ * As Java's implementation of multidimensional arrays is not memory efficient,
+ * this implementation uses indirection where one array designates the starting
+ * and ending entry in another array.
+ *
+ * The memory usage for 10M documents with 5M unique tags with 9M references in
+ * facet A, 15M unique tags with 15M references in facet B and 10 unique tags
+ * with 20M references in facet C is thus {@code
+(9M + 15M + 20M) * log2(5M + 15M + 10) bits = for tag references
+10M * log2(9M + 15M + 20M) bits for the doc2tagref array
+5M * log2(5M) + 15M * log2(15M) + 10 * log2(10) bits TermProviders
+} which gives a total of 220 MB. Depending on setup, sorting and the need for
+ * fast re-opening of the index, an extra copy of the TermProvider-data might
+ * be needed, bringing the total memory usage up to 275 MB. In order to actually
+ * use the structure for faceting, a counting array is needed. This is an int[]
+ * with an entry for each unique tag (20000010 or about 20M in this example).
+ * This adds an extra 80 MB for each concurrent thread.
+ *
+ * There is some temporary memory overhead while building the structures, but
+ * this is less than the total size of the final structures so a rule of thumb
+ * is to double that size to get the upper bound. Thus 550 MB of heap is
+ * required for the whole shebang, which leaves (550-275)/80MB ~= 3 concurrent
+ * faceting searches.
+ *
+ * Scaling down, 1M documents with a single facet with 500K unique tags and
+ * 1M occurrences takes about 20 MB with 3 concurrent searches.
+ *
+ * Scaling up, 100M documents with 3 facets with 100M unique tags and
+ * 1000M occurrences each can be done on a 24GB machine. However, counting
+ * the tags from a search result would take a while.
+ */
+public class ExposedFacets {
+ /**
+ * The maximum number of collectors to hold in the collector pool.
+ * Each collector takes up 4 * #documents bytes in the index.
+ */
+ private static final int MAX_COLLECTORS = 10;
+
+ private CollectorPool collectorPool;
+
+ public void facet(FacetRequest request, IndexSearcher searcher, Query query)
+ throws IOException {
+ TagCollector collector = collectorPool.acquire(request.getQuery());
+ try {
+ searcher.search(query, collector);
+ } finally {
+ collectorPool.release(request.getQuery(), collector);
+ }
+ }
+
+
+ // TODO: Support match-all without search
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java Sat Sep 18 01:19:34 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java Sat Sep 18 01:19:34 CEST 2010
@@ -0,0 +1,214 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+public class TestFieldTermProvider extends LuceneTestCase {
+// new File(System.getProperty("java.io.tmpdir"), "exposed_index");
+// new File("/home/te/projects/lucene/exposed_index");
+// new File("/mnt/bulk/exposed_index");
+ public static final int DOCCOUNT = 100;
+ private ExposedHelper helper;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ // TODO: Figure out how to force Flex the clean way
+ CodecProvider.setDefaultCodec("Standard");
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ helper.close();
+ }
+
+ public void testCreateIndex() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ }
+
+ public void testSegmentcount() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ int subCount = reader.getSequentialSubReaders().length;
+ assertTrue("The number of segments should be >= 2 but was " + subCount,
+ subCount >= 2);
+ reader.close();
+ }
+
+ public void testIndexGeneration() throws Exception {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ long termCount = 0;
+ TermsEnum terms = MultiFields.getFields(reader).
+ terms(ExposedHelper.ID).iterator();
+ while (terms.next() != null) {
+ assertEquals("The ID-term #" + termCount + " should be correct",
+ ExposedHelper.ID_FORMAT.format(termCount),
+ terms.term().utf8ToString());
+ termCount++;
+ }
+
+ assertEquals("There should be the right number of terms",
+ DOCCOUNT, termCount);
+ reader.close();
+ }
+
+ public void testOrdinalAccess() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ TermsEnum terms =
+ reader.getSequentialSubReaders()[0].fields().terms("a").iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ }
+
+ TermProvider segmentProvider = ExposedFactory.createProvider(
+ segment, null, Arrays.asList("a"), null, "null");
+ ExposedRequest.Field request = new ExposedRequest.Field("a", null, "null");
+ ArrayList exposedOrdinals = new ArrayList(DOCCOUNT);
+ for (int i = 0 ; i < segmentProvider.getOrdinalTermCount() ; i++) {
+ exposedOrdinals.add(segmentProvider.getTerm(i).utf8ToString());
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedOrdinals.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedOrdinals.get(i));
+ }
+ reader.close();
+
+ }
+
+ public void testSegmentTermSort() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+ testSegmentTermSort(segment);
+ reader.close();
+ }
+
+ private void testSegmentTermSort(IndexReader segment) throws IOException {
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ TermsEnum terms = segment.fields().terms("a").iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ System.out.println("Unsorted term #" + (plainExtraction.size() - 1) + ": "
+ + terms.term().utf8ToString());
+ }
+ Collections.sort(plainExtraction, sorter);
+
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(sorter), "collator_da");
+ FieldTermProvider segmentProvider =
+ new FieldTermProvider(segment, request, true);
+
+ ArrayList exposedExtraction = new ArrayList(DOCCOUNT);
+ Iterator ei = segmentProvider.getIterator(false);
+ int count = 0;
+ while (ei.hasNext()) {
+ exposedExtraction.add(ei.next().term.utf8ToString());
+ System.out.println("Exposed sorted term #" + count++ + ": "
+ + exposedExtraction.get(exposedExtraction.size()-1));
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedExtraction.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedExtraction.get(i));
+ }
+ }
+
+ public void testDocIDMapping() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plain =
+ new ArrayList(DOCCOUNT);
+ for (int docID = 0 ; docID < segment.maxDoc() ; docID++) {
+ plain.add(new ExposedHelper.Pair(
+ docID, segment.document(docID).get("a"), sorter));
+// System.out.println("Plain access added " + plain.get(plain.size()-1));
+ }
+ Collections.sort(plain);
+
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(sorter), "collator_da");
+ FieldTermProvider segmentProvider =
+ new FieldTermProvider(segment, request, true);
+
+ ArrayList exposed =
+ new ArrayList(DOCCOUNT);
+ Iterator ei = segmentProvider.getIterator(true);
+
+ while (ei.hasNext()) {
+ ExposedTuple next = ei.next();
+ exposed.add(new ExposedHelper.Pair(
+ next.docID, next.term.utf8ToString(), sorter));
+ }
+ Collections.sort(exposed);
+
+ assertEquals("The two docID->term maps should be of equal length",
+ plain.size(), exposed.size());
+ for (int i = 0 ; i < plain.size() ; i++) {
+ assertEquals("Mapping #" + i + " should be equal but was "
+ + plain.get(i) + " vs. " + exposed.get(i),
+ plain.get(i), exposed.get(i));
+ System.out.println("Sorted docID, term #" + i + ": " + plain.get(i));
+ }
+ reader.close();
+ }
+
+ // Just tests for non-crashing
+ public void testDocIDMultiMapping() throws IOException {
+ helper.createIndex(100, Arrays.asList("a"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ ExposedHelper.MULTI,
+ ExposedComparators.collatorToBytesRef(sorter), "collator_da");
+ FieldTermProvider segmentProvider =
+ new FieldTermProvider(segment, request, true);
+
+ ArrayList exposed =
+ new ArrayList(DOCCOUNT);
+ Iterator ei = segmentProvider.getIterator(true);
+
+ while (ei.hasNext()) {
+ ExposedTuple next = ei.next();
+ exposed.add(new ExposedHelper.Pair(
+ next.docID, next.term.utf8ToString(), sorter));
+ }
+
+/* for (int i = 0 ; i < exposed.size() ; i++) {
+ System.out.println("Sorted docID, term #" + i + ": " + exposed.get(i));
+ }*/
+ reader.close();
+ }
+
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java Sun Sep 19 20:42:32 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.java Sun Sep 19 20:42:32 CEST 2010
@@ -0,0 +1,183 @@
+package org.apache.lucene.search.exposed.facet;
+
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.StringWriter;
+import java.util.List;
+
+public class FacetResponse {
+ public static final String NAMESPACE =
+ "http://lucene.apache.org/exposed/facet/response/1.0";
+
+ private static final XMLOutputFactory xmlOutFactory;
+ static {
+ xmlOutFactory = XMLOutputFactory.newFactory();
+ }
+
+ private final FacetRequest request;
+ private final List groups;
+ private long hits;
+ private long countingTime = -1; // ms: just counting the references
+ private long totalTime = -1; // ms
+ private boolean countCached = false;
+
+ public FacetResponse(FacetRequest request, List groups, long hits) {
+ this.request = request;
+ this.groups = groups;
+ this.hits = hits;
+ }
+
+ public void setTotalTime(long totalTime) {
+ this.totalTime = totalTime;
+ }
+
+ public void setCountingTime(long countingTime) {
+ this.countingTime = countingTime;
+ }
+
+ public String toXML() {
+ StringWriter sw = new StringWriter();
+ try {
+ XMLStreamWriter out = xmlOutFactory.createXMLStreamWriter(sw);
+ out.writeStartDocument("utf-8", "1.0");
+ out.writeCharacters("\n");
+ toXML(out);
+ out.writeEndDocument();
+ out.flush();
+ } catch (XMLStreamException e) { // Getting here means error in the code
+ throw new RuntimeException("Unable to create XML", e);
+ }
+ return sw.toString();
+ }
+ public void toXML(XMLStreamWriter out) throws XMLStreamException {
+ out.setDefaultNamespace(NAMESPACE);
+ out.writeStartElement("facetresponse");
+ out.writeDefaultNamespace(NAMESPACE);
+ out.writeAttribute("query", request.getQuery());
+ writeIfDefined(out, "hits", hits);
+ writeIfDefined(out, "countms", countingTime);
+ out.writeAttribute("countcached", countCached ? "true" : "false");
+ writeIfDefined(out, "totalms", totalTime);
+ out.writeCharacters("\n");
+ for (Group group: groups) {
+ group.toXML(out);
+ }
+ out.writeEndElement(); //
+ out.writeCharacters("\n");
+ }
+
+ public static class Group {
+ private final FacetRequest.FacetGroup request;
+ private final List tags;
+ private long potentialTags = -1;
+ private long totalReferences = -1;
+ private long validTags = -1;
+ private long extractionTime = -1; // ms
+
+ public Group(FacetRequest.FacetGroup request, List tags) {
+ this.request = request;
+ this.tags = tags;
+ }
+
+ public void toXML(XMLStreamWriter out) throws XMLStreamException {
+ out.writeCharacters(" ");
+ out.writeStartElement("facet");
+ out.writeAttribute("name", request.getGroup().getName());
+ out.writeAttribute("fields", getFields());
+ out.writeAttribute("order", request.getOrder());
+ if (request.getLocale() != null) {
+ out.writeAttribute("locale", request.getLocale());
+ }
+ writeIfDefined(out, "maxtags", request.getMaxTags());
+ writeIfDefined(out, "mincount", request.getMinCount());
+ writeIfDefined(out, "offset", request.getOffset());
+
+ writeIfDefined(out, "prefix", request.getPrefix());
+
+ writeIfDefined(out, "potentialtags", potentialTags);
+ writeIfDefined(out, "usedreferences", totalReferences);
+ writeIfDefined(out, "validtags", validTags);
+ writeIfDefined(out, "extractionms", extractionTime);
+ out.writeCharacters("\n");
+
+ for (Tag tag: tags) {
+ tag.toXML(out);
+ }
+
+ out.writeCharacters(" ");
+ out.writeEndElement(); //
+ out.writeCharacters("\n");
+ }
+
+ private StringBuffer sb = new StringBuffer();
+ private synchronized String getFields() {
+ sb.setLength(0);
+ List fieldNames = request.getGroup().getFieldNames();
+ for (int i = 0 ; i < fieldNames.size() ; i++) {
+ if (i != 0) {
+ sb.append(", ");
+ }
+ sb.append(fieldNames.get(i));
+ }
+ return sb.toString();
+ }
+
+ public void setPotentialTags(long potentialTags) {
+ this.potentialTags = potentialTags;
+ }
+ public void setTotalReferences(long usedReferences) {
+ this.totalReferences = usedReferences;
+ }
+ public void setValidTags(long validTags) {
+ this.validTags = validTags;
+ }
+ public void setExtractionTime(long extractionTime) {
+ this.extractionTime = extractionTime;
+ }
+ }
+
+ public static class Tag {
+ private final String term;
+ private final int count;
+
+ public Tag(String term, int count) {
+ this.term = term;
+ this.count = count;
+ }
+
+ public void toXML(XMLStreamWriter out) throws XMLStreamException {
+ out.writeCharacters(" ");
+ out.writeStartElement("tag");
+ out.writeAttribute("count", Integer.toString(count));
+ out.writeCharacters(term);
+ out.writeEndElement();
+ out.writeCharacters("\n");
+ }
+ }
+
+ private static void writeIfDefined(XMLStreamWriter out, String attributeName,
+ long value) throws XMLStreamException {
+ if (value == -1) {
+ return;
+ }
+ out.writeAttribute(attributeName, Long.toString(value));
+ }
+
+ private static void writeIfDefined(XMLStreamWriter out, String attributeName,
+ String value) throws XMLStreamException {
+ if (value == null || "".equals(value)) {
+ return;
+ }
+ out.writeAttribute(attributeName, value);
+ }
+
+ /**
+ * @param wasCached true if the counting part of the facet generation was
+ * cached.
+ */
+ public void setCountingCached(boolean wasCached) {
+ this.countCached = wasCached;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java Thu Aug 12 13:57:51 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java Thu Aug 12 13:57:51 CEST 2010
@@ -0,0 +1,103 @@
+package org.apache.lucene.search.exposed.poc;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty;
+import static org.apache.lucene.util.automaton.BasicAutomata.makeString;
+
+/**
+ * A tokenfilter for testing that removes terms accepted by a DFA.
+ *
+ * - Union a list of singletons to act like a stopfilter.
+ *
- Use the complement to act like a keepwordfilter
+ *
- Use a regex like
.{12,} to act like a lengthfilter
+ *
+ */
+public final class MockTokenFilter extends TokenFilter {
+ /** Empty set of stopwords */
+ public static final CharacterRunAutomaton EMPTY_STOPSET =
+ new CharacterRunAutomaton(makeEmpty());
+
+ /** Set of common english stopwords */
+ public static final CharacterRunAutomaton ENGLISH_STOPSET =
+ new CharacterRunAutomaton(BasicOperations.union(Arrays.asList(
+ makeString("a"), makeString("an"), makeString("and"), makeString("are"),
+ makeString("as"), makeString("at"), makeString("be"), makeString("but"),
+ makeString("by"), makeString("for"), makeString("if"), makeString("in"),
+ makeString("into"), makeString("is"), makeString("it"), makeString("no"),
+ makeString("not"), makeString("of"), makeString("on"), makeString("or"),
+ makeString("such"), makeString("that"), makeString("the"), makeString("their"),
+ makeString("then"), makeString("there"), makeString("these"), makeString("they"),
+ makeString("this"), makeString("to"), makeString("was"), makeString("will"),
+ makeString("with"))));
+
+ private final CharacterRunAutomaton filter;
+ private boolean enablePositionIncrements = false;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ super(input);
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ // return the first non-stop word found
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (!filter.run(termAtt.buffer(), 0, termAtt.length())) {
+ if (enablePositionIncrements) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+ return true;
+ }
+ skippedPositions += posIncrAtt.getPositionIncrement();
+ }
+ // reached EOS -- return false
+ return false;
+ }
+
+ /**
+ * @see #setEnablePositionIncrements(boolean)
+ */
+ public boolean getEnablePositionIncrements() {
+ return enablePositionIncrements;
+ }
+
+ /**
+ * If true, this Filter will preserve
+ * positions of the incoming tokens (ie, accumulate and
+ * set position increments of the removed stop tokens).
+ */
+ public void setEnablePositionIncrements(boolean enable) {
+ this.enablePositionIncrements = enable;
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java Mon Sep 20 16:17:41 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java Mon Sep 20 16:17:41 CEST 2010
@@ -0,0 +1,225 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Multi source Iterator that merges the ExposedTuples according to the given
+ * comparator. Designed to work tightly with {@link GroupTermProvider}.
+ *
+ * It is recommended to use {@link CachedTermProvider} with look-ahead on the
+ * sources or otherwise ensure that ordered iteration is fast if no docIDs are
+ * to be returned. If docIDs are to be returned, caching at the source level
+ * will not help.
+ *
+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless
+ * {@link #setReuseTuple(boolean)} is called with false.
+ *
+ * The merger works by keeping a list of the first elements from all sources
+ * and seeding a priority queue with indexes 0...#sources-1. When the PQ is
+ * popped, the elements corresponding to the indexes in the PQ are compared.
+ * When an element is returned, it is delivered to the iterator. The next time
+ * next() is called on the iterator, the index for the previous tuple is
+ * re-inserted into the PQ, unless the source for that index is empty.
+ * This ensures full reuse of the tuples and thus low impact on GC.
+ */
+class MergingTermDocIterator implements Iterator {
+ private final List> iterators;
+ private final boolean collectDocIDs;
+ private final GroupTermProvider groupProvider;
+
+ private final ExposedTuple[] backingTuples;
+ private final ExposedPriorityQueue pq;
+
+ private final Collator collator;
+ private final CollationKey[] backingKeys; // Only used with Collator
+
+ // TODO: Make reuseTuple = true work
+ private boolean reuseTuple = false;
+ private long indirect = 0;
+
+ /*
+ If pending is true, the currentIndex points to the iterator and backingTuple
+ that is to be drained.
+ */
+ private boolean pending = false; // true == Terms or docIDs are ready in tuple
+ /**
+ * If pending is true, this is ready for delivery.
+ */
+ private final ExposedTuple tuple =
+ new ExposedTuple("", new BytesRef(), -1, -1, -1);
+ /**
+ * If pending is true, this is the index for the source responsible for
+ * delivering the tuple.
+ */
+ private int currentIndex = -1;
+
+ /**
+ * If this is true and the given comparator is a
+ * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized
+ * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the
+ * chunk-sort speed.
+ */
+ public static boolean optimizeCollator = true;
+
+ public MergingTermDocIterator(
+ GroupTermProvider groupProvider, List sources,
+ Comparator comparator, boolean collectDocIDs)
+ throws IOException {
+ this.groupProvider = groupProvider;
+ this.collectDocIDs = collectDocIDs;
+
+ iterators = new ArrayList>(sources.size());
+ backingTuples = new ExposedTuple[sources.size()];
+
+ collator = optimizeCollator &&
+ comparator instanceof ExposedComparators.BytesRefWrappedCollator
+ ? ((ExposedComparators.BytesRefWrappedCollator)comparator).getCollator()
+ : null;
+ backingKeys = new CollationKey[sources.size()];
+
+ ExposedComparators.OrdinalComparator wrappedComparator =
+ collator == null ?
+ ExposedComparators.wrapBacking(backingTuples, comparator)
+ : ExposedComparators.wrapBacking(backingKeys);
+ pq = new ExposedPriorityQueue(wrappedComparator, sources.size());
+ int index = 0;
+ for (TermProvider source: sources) {
+ // TODO: Divide cacheLimit and avoid cache on collectDocIDs(?)
+ Iterator iterator = source.getIterator(collectDocIDs);
+ iterators.add(iterator);
+ ExposedTuple tuple = getNextTuple(index);
+ if (tuple != null) {
+ backingTuples[index] = tuple;
+ if (collator != null) {
+ backingKeys[index] =
+ collator.getCollationKey(tuple.term.utf8ToString());
+ }
+ pq.add(index);
+ }
+ index++;
+ }
+ }
+
+ // Ensure that we can deliver an id
+ public boolean hasNext() {
+ while (true) {
+ if (pending) {
+// System.out.println("hasnext ***" + tuple.term.utf8ToString());
+ return true;
+ }
+ if (pq.size() == 0) {
+ return false;
+ }
+ try {
+ seekToNextTuple();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next docID", e);
+ }
+ }
+ }
+
+ public ExposedTuple next() {
+// System.out.println("next " + tuple.term.utf8ToString());
+ if (!hasNext()) {
+ throw new IllegalStateException("The iterator is depleted");
+ }
+ ExposedTuple delivery = reuseTuple ? tuple : new ExposedTuple(tuple);
+ pending = false;
+// System.out.println("Merging delivery: " + delivery);
+ return delivery;
+ }
+
+ // Pops a tuple from the iterator indicated by the index and modifies term
+ // ordinal and docID from segment to index level.
+ // If index == -1 or hasNext() == false, null is returned.
+ private ExposedTuple getNextTuple(int index) throws IOException {
+ if (index == -1 || !iterators.get(index).hasNext()) {
+ return null;
+ }
+ // TODO: Avoid the creation of a new tuple by assigning to existing tuple
+ ExposedTuple nextTuple = new ExposedTuple(iterators.get(index).next());
+ nextTuple.docID = groupProvider.segmentToIndexDocID(
+ index, nextTuple.docID);
+ nextTuple.ordinal = groupProvider.segmentToIndexTermOrdinal(
+ index, nextTuple.ordinal);
+ nextTuple.indirect = -1; // Just for good measure: It will be updated later
+ return nextTuple;
+ }
+
+ private void seekToNextTuple() throws IOException {
+
+/*
+ if (collectDocIDs && currentIndex != -1) {
+
+ // Start by checking if the previous source has more tuples with the
+ // same ordinal (e.g. if it has more docIDs)
+ ExposedTuple candidate = backingTuples[currentIndex];
+ if (candidate != null && candidate.ordinal == tuple.ordinal) {
+ // Field, term, ordinal and indirect are the same
+ tuple.docID = candidate.docID;
+ pending = true;
+ backingTuples[currentIndex] = getNextTuple(currentIndex);
+ return;
+ }
+ }
+ */
+ // DocID-optimization must take place before reinserting into pq
+
+ // Take the next tuple
+ while (!pending && pq.size() > 0) {
+ final ExposedTuple newTuple = pop();
+ if (newTuple == null) {
+ break;
+ }
+
+ if (tuple.term.equals(newTuple.term)) {
+ if (!collectDocIDs) {
+ continue; // Skip duplicates whan no docID
+ }
+ tuple.set(newTuple.field, tuple.term, tuple.ordinal, tuple.indirect,
+ newTuple.docID);
+ } else {
+ tuple.set(newTuple.field, newTuple.term, newTuple.ordinal, indirect++,
+ newTuple.docID);
+ }
+ pending = true;
+ }
+ }
+
+ private ExposedTuple pop() throws IOException {
+ currentIndex = pq.pop();
+ if (currentIndex == -1) {
+ return null;
+ }
+ final ExposedTuple foundTuple = backingTuples[currentIndex];
+
+ final ExposedTuple newTuple = getNextTuple(currentIndex);
+ if (newTuple != null) {
+ backingTuples[currentIndex] = newTuple;
+ if (collator != null) {
+ backingKeys[currentIndex] =
+ collator.getCollationKey(newTuple.term.utf8ToString());
+ }
+ pq.add(currentIndex);
+ }
+ return foundTuple;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("Not a valid operation");
+ }
+
+ public void setReuseTuple(boolean reuseTuple) {
+ this.reuseTuple = reuseTuple;
+ }
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,920 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.exposed;
+
+import java.util.Arrays;
+
+/**
+ * A stable, adaptive, iterative mergesort that requires far fewer than
+ * n lg(n) comparisons when running on partially sorted arrays, while
+ * offering performance comparable to a traditional mergesort when run
+ * on random arrays. Like all proper mergesorts, this sort is stable and
+ * runs O(n log n) time (worst case). In the worst case, this sort requires
+ * temporary storage space for n/2 object references; in the best case,
+ * it requires only a small constant amount of space.
+ *
+ * This implementation was adapted from Tim Peters's list sort for
+ * Python, which is described in detail here:
+ *
+ * http://svn.python.org/projects/python/trunk/Objects/listsort.txt
+ *
+ * Tim's C code may be found here:
+ *
+ * http://svn.python.org/projects/python/trunk/Objects/listobject.c
+ *
+ * The underlying techniques are described in this paper (and may have
+ * even earlier origins):
+ *
+ * "Optimistic Sorting and Information Theoretic Complexity"
+ * Peter McIlroy
+ * SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms),
+ * pp 467-474, Austin, Texas, 25-27 January 1993.
+ *
+ * While the API to this class consists solely of static methods, it is
+ * (privately) instantiable; a TimSort instance holds the state of an ongoing
+ * sort, assuming the input array is large enough to warrant the full-blown
+ * TimSort. Small arrays are sorted in place, using a binary insertion sort.
+ *
+ * @author Josh Bloch
+ *
+ * Modified 2010-03-22 to handle int only (specialization to save memory)
+ * by Toke Eskildsen (te@statsbiblioteket.dk).
+ */
+public class ExposedTimSort {
+ /**
+ * This is the minimum sized sequence that will be merged. Shorter
+ * sequences will be lengthened by calling binarySort. If the entire
+ * array is less than this length, no merges will be performed.
+ *
+ * This constant should be a power of two. It was 64 in Tim Peter's C
+ * implementation, but 32 was empirically determined to work better in
+ * this implementation. In the unlikely event that you set this constant
+ * to be a number that's not a power of two, you'll need to change the
+ * {@link #minRunLength} computation.
+ *
+ * If you decrease this constant, you must change the stackLen
+ * computation in the TimSort constructor, or you risk an
+ * ArrayOutOfBounds exception. See listsort.txt for a discussion
+ * of the minimum stack length required as a function of the length
+ * of the array being sorted and the minimum merge sequence length.
+ */
+ private static final int MIN_MERGE = 32;
+
+ /**
+ * The array being sorted.
+ */
+ private final int[] a;
+
+ /**
+ * The comparator for this sort.
+ */
+ private final ExposedComparators.OrdinalComparator c;
+
+ /**
+ * When we get into galloping mode, we stay there until both runs win less
+ * often than MIN_GALLOP consecutive times.
+ */
+ private static final int MIN_GALLOP = 7;
+
+ /**
+ * This controls when we get *into* galloping mode. It is initialized
+ * to MIN_GALLOP. The mergeLo and mergeHi methods nudge it higher for
+ * random data, and lower for highly structured data.
+ */
+ private int minGallop = MIN_GALLOP;
+
+ /**
+ * Maximum initial size of tmp array, which is used for merging. The array
+ * can grow to accommodate demand.
+ *
+ * Unlike Tim's original C version, we do not allocate this much storage
+ * when sorting smaller arrays. This change was required for performance.
+ */
+ private static final int INITIAL_TMP_STORAGE_LENGTH = 256;
+
+ /**
+ * Temp storage for merges.
+ */
+ private int[] tmp; // Actual runtime type will be Object[], regardless of T
+
+ /**
+ * A stack of pending runs yet to be merged. Run i starts at
+ * address base[i] and extends for len[i] elements. It's always
+ * true (so long as the indices are in bounds) that:
+ *
+ * runBase[i] + runLen[i] == runBase[i + 1]
+ *
+ * so we could cut the storage for this, but it's a minor amount,
+ * and keeping all the info explicit simplifies the code.
+ */
+ private int stackSize = 0; // Number of pending runs on stack
+ private final int[] runBase;
+ private final int[] runLen;
+
+ /**
+ * Creates a TimSort instance to maintain the state of an ongoing sort.
+ *
+ * @param a the array to be sorted
+ * @param c the comparator to determine the order of the sort
+ */
+ private ExposedTimSort(int[] a, ExposedComparators.OrdinalComparator c) {
+ this.a = a;
+ this.c = c;
+
+ // Allocate temp storage (which may be increased later if necessary)
+ int len = a.length;
+ tmp = new int[len < 2 * INITIAL_TMP_STORAGE_LENGTH ?
+ len >>> 1 : INITIAL_TMP_STORAGE_LENGTH];
+
+ /*
+ * Allocate runs-to-be-merged stack (which cannot be expanded). The
+ * stack length requirements are described in listsort.txt. The C
+ * version always uses the same stack length (85), but this was
+ * measured to be too expensive when sorting "mid-sized" arrays (e.g.,
+ * 100 elements) in Java. Therefore, we use smaller (but sufficiently
+ * large) stack lengths for smaller arrays. The "magic numbers" in the
+ * computation below must be changed if MIN_MERGE is decreased. See
+ * the MIN_MERGE declaration above for more information.
+ */
+ int stackLen = (len < 120 ? 5 :
+ len < 1542 ? 10 :
+ len < 119151 ? 19 : 40);
+ runBase = new int[stackLen];
+ runLen = new int[stackLen];
+ }
+
+ /*
+ * The next two methods (which are package private and static) constitute
+ * the entire API of this class. Each of these methods obeys the contract
+ * of the public method with the same signature in java.util.Arrays.
+ */
+
+ public static void sort(int[] a, ExposedComparators.OrdinalComparator c) {
+ sort(a, 0, a.length, c);
+ }
+
+ public static void sort(int[] a, int lo, int hi, ExposedComparators.OrdinalComparator c) {
+ if (c == null) {
+ Arrays.sort(a, lo, hi);
+ return;
+ }
+
+ rangeCheck(a.length, lo, hi);
+ int nRemaining = hi - lo;
+ if (nRemaining < 2)
+ return; // Arrays of size 0 and 1 are always sorted
+
+ // If array is small, do a "mini-TimSort" with no merges
+ if (nRemaining < MIN_MERGE) {
+ int initRunLen = countRunAndMakeAscending(a, lo, hi, c);
+ binarySort(a, lo, hi, lo + initRunLen, c);
+ return;
+ }
+
+ /**
+ * March over the array once, left to right, finding natural runs,
+ * extending short natural runs to minRun elements, and merging runs
+ * to maintain stack invariant.
+ */
+ ExposedTimSort ts = new ExposedTimSort(a, c);
+ int minRun = minRunLength(nRemaining);
+ do {
+ // Identify next run
+ int runLen = countRunAndMakeAscending(a, lo, hi, c);
+
+ // If run is short, extend to min(minRun, nRemaining)
+ if (runLen < minRun) {
+ int force = nRemaining <= minRun ? nRemaining : minRun;
+ binarySort(a, lo, lo + force, lo + runLen, c);
+ runLen = force;
+ }
+
+ // Push run onto pending-run stack, and maybe merge
+ ts.pushRun(lo, runLen);
+ ts.mergeCollapse();
+
+ // Advance to find next run
+ lo += runLen;
+ nRemaining -= runLen;
+ } while (nRemaining != 0);
+
+ // Merge all remaining runs to complete sort
+ assert lo == hi;
+ ts.mergeForceCollapse();
+ assert ts.stackSize == 1;
+ }
+
+ /**
+ * Sorts the specified portion of the specified array using a binary
+ * insertion sort. This is the best method for sorting small numbers
+ * of elements. It requires O(n log n) compares, but O(n^2) data
+ * movement (worst case).
+ *
+ * If the initial part of the specified range is already sorted,
+ * this method can take advantage of it: the method assumes that the
+ * elements from index {@code lo}, inclusive, to {@code start},
+ * exclusive are already sorted.
+ *
+ * @param a the array in which a range is to be sorted
+ * @param lo the index of the first element in the range to be sorted
+ * @param hi the index after the last element in the range to be sorted
+ * @param start the index of the first element in the range that is
+ * not already known to be sorted (@code lo <= start <= hi}
+ * @param c comparator to used for the sort
+ */
+ @SuppressWarnings("fallthrough")
+ private static void binarySort(int[] a, int lo, int hi, int start,
+ ExposedComparators.OrdinalComparator c) {
+ assert lo <= start && start <= hi;
+ if (start == lo)
+ start++;
+ for ( ; start < hi; start++) {
+ int pivot = a[start];
+
+ // Set left (and right) to the index where a[start] (pivot) belongs
+ int left = lo;
+ int right = start;
+ assert left <= right;
+ /*
+ * Invariants:
+ * pivot >= all in [lo, left).
+ * pivot < all in [right, start).
+ */
+ while (left < right) {
+ int mid = (left + right) >>> 1;
+ if (c.compare(pivot, a[mid]) < 0)
+ right = mid;
+ else
+ left = mid + 1;
+ }
+ assert left == right;
+
+ /*
+ * The invariants still hold: pivot >= all in [lo, left) and
+ * pivot < all in [left, start), so pivot belongs at left. Note
+ * that if there are elements equal to pivot, left points to the
+ * first slot after them -- that's why this sort is stable.
+ * Slide elements over to make room to make room for pivot.
+ */
+ int n = start - left; // The number of elements to move
+ // Switch is just an optimization for arraycopy in default case
+ switch(n) {
+ case 2: a[left + 2] = a[left + 1];
+ case 1: a[left + 1] = a[left];
+ break;
+ default: System.arraycopy(a, left, a, left + 1, n);
+ }
+ a[left] = pivot;
+ }
+ }
+
+ /**
+ * Returns the length of the run beginning at the specified position in
+ * the specified array and reverses the run if it is descending (ensuring
+ * that the run will always be ascending when the method returns).
+ *
+ * A run is the longest ascending sequence with:
+ *
+ * a[lo] <= a[lo + 1] <= a[lo + 2] <= ...
+ *
+ * or the longest descending sequence with:
+ *
+ * a[lo] > a[lo + 1] > a[lo + 2] > ...
+ *
+ * For its intended use in a stable mergesort, the strictness of the
+ * definition of "descending" is needed so that the call can safely
+ * reverse a descending sequence without violating stability.
+ *
+ * @param a the array in which a run is to be counted and possibly reversed
+ * @param lo index of the first element in the run
+ * @param hi index after the last element that may be contained in the run.
+ It is required that @code{lo < hi}.
+ * @param c the comparator to used for the sort
+ * @return the length of the run beginning at the specified position in
+ * the specified array
+ */
+ private static int countRunAndMakeAscending(int[] a, int lo, int hi,
+ ExposedComparators.OrdinalComparator c) {
+ assert lo < hi;
+ int runHi = lo + 1;
+ if (runHi == hi)
+ return 1;
+
+ // Find end of run, and reverse range if descending
+ if (c.compare(a[runHi++], a[lo]) < 0) { // Descending
+ while(runHi < hi && c.compare(a[runHi], a[runHi - 1]) < 0)
+ runHi++;
+ reverseRange(a, lo, runHi);
+ } else { // Ascending
+ while (runHi < hi && c.compare(a[runHi], a[runHi - 1]) >= 0)
+ runHi++;
+ }
+
+ return runHi - lo;
+ }
+
+ /**
+ * Reverse the specified range of the specified array.
+ *
+ * @param a the array in which a range is to be reversed
+ * @param lo the index of the first element in the range to be reversed
+ * @param hi the index after the last element in the range to be reversed
+ */
+ private static void reverseRange(int[] a, int lo, int hi) {
+ hi--;
+ while (lo < hi) {
+ int t = a[lo];
+ a[lo++] = a[hi];
+ a[hi--] = t;
+ }
+ }
+
+ /**
+ * Returns the minimum acceptable run length for an array of the specified
+ * length. Natural runs shorter than this will be extended with
+ * {@link #binarySort}.
+ *
+ * Roughly speaking, the computation is:
+ *
+ * If n < MIN_MERGE, return n (it's too small to bother with fancy stuff).
+ * Else if n is an exact power of 2, return MIN_MERGE/2.
+ * Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k
+ * is close to, but strictly less than, an exact power of 2.
+ *
+ * For the rationale, see listsort.txt.
+ *
+ * @param n the length of the array to be sorted
+ * @return the length of the minimum run to be merged
+ */
+ private static int minRunLength(int n) {
+ assert n >= 0;
+ int r = 0; // Becomes 1 if any 1 bits are shifted off
+ while (n >= MIN_MERGE) {
+ r |= (n & 1);
+ n >>= 1;
+ }
+ return n + r;
+ }
+
+ /**
+ * Pushes the specified run onto the pending-run stack.
+ *
+ * @param runBase index of the first element in the run
+ * @param runLen the number of elements in the run
+ */
+ private void pushRun(int runBase, int runLen) {
+ this.runBase[stackSize] = runBase;
+ this.runLen[stackSize] = runLen;
+ stackSize++;
+ }
+
+ /**
+ * Examines the stack of runs waiting to be merged and merges adjacent runs
+ * until the stack invariants are reestablished:
+ *
+ * 1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1]
+ * 2. runLen[i - 2] > runLen[i - 1]
+ *
+ * This method is called each time a new run is pushed onto the stack,
+ * so the invariants are guaranteed to hold for i < stackSize upon
+ * entry to the method.
+ */
+ private void mergeCollapse() {
+ while (stackSize > 1) {
+ int n = stackSize - 2;
+ if (n > 0 && runLen[n-1] <= runLen[n] + runLen[n+1]) {
+ if (runLen[n - 1] < runLen[n + 1])
+ n--;
+ mergeAt(n);
+ } else if (runLen[n] <= runLen[n + 1]) {
+ mergeAt(n);
+ } else {
+ break; // Invariant is established
+ }
+ }
+ }
+
+ /**
+ * Merges all runs on the stack until only one remains. This method is
+ * called once, to complete the sort.
+ */
+ private void mergeForceCollapse() {
+ while (stackSize > 1) {
+ int n = stackSize - 2;
+ if (n > 0 && runLen[n - 1] < runLen[n + 1])
+ n--;
+ mergeAt(n);
+ }
+ }
+
+ /**
+ * Merges the two runs at stack indices i and i+1. Run i must be
+ * the penultimate or antepenultimate run on the stack. In other words,
+ * i must be equal to stackSize-2 or stackSize-3.
+ *
+ * @param i stack index of the first of the two runs to merge
+ */
+ private void mergeAt(int i) {
+ assert stackSize >= 2;
+ assert i >= 0;
+ assert i == stackSize - 2 || i == stackSize - 3;
+
+ int base1 = runBase[i];
+ int len1 = runLen[i];
+ int base2 = runBase[i + 1];
+ int len2 = runLen[i + 1];
+ assert len1 > 0 && len2 > 0;
+ assert base1 + len1 == base2;
+
+ /*
+ * Record the length of the combined runs; if i is the 3rd-last
+ * run now, also slide over the last run (which isn't involved
+ * in this merge). The current run (i+1) goes away in any case.
+ */
+ runLen[i] = len1 + len2;
+ if (i == stackSize - 3) {
+ runBase[i + 1] = runBase[i + 2];
+ runLen[i + 1] = runLen[i + 2];
+ }
+ stackSize--;
+
+ /*
+ * Find where the first element of run2 goes in run1. Prior elements
+ * in run1 can be ignored (because they're already in place).
+ */
+ int k = gallopRight(a[base2], a, base1, len1, 0, c);
+ assert k >= 0;
+ base1 += k;
+ len1 -= k;
+ if (len1 == 0)
+ return;
+
+ /*
+ * Find where the last element of run1 goes in run2. Subsequent elements
+ * in run2 can be ignored (because they're already in place).
+ */
+ len2 = gallopLeft(a[base1 + len1 - 1], a, base2, len2, len2 - 1, c);
+ assert len2 >= 0;
+ if (len2 == 0)
+ return;
+
+ // Merge remaining runs, using tmp array with min(len1, len2) elements
+ if (len1 <= len2)
+ mergeLo(base1, len1, base2, len2);
+ else
+ mergeHi(base1, len1, base2, len2);
+ }
+
+ /**
+ * Locates the position at which to insert the specified key into the
+ * specified sorted range; if the range contains an element equal to key,
+ * returns the index of the leftmost equal element.
+ *
+ * @param key the key whose insertion point to search for
+ * @param a the array in which to search
+ * @param base the index of the first element in the range
+ * @param len the length of the range; must be > 0
+ * @param hint the index at which to begin the search, 0 <= hint < n.
+ * The closer hint is to the result, the faster this method will run.
+ * @param c the comparator used to order the range, and to search
+ * @return the int k, 0 <= k <= n such that a[b + k - 1] < key <= a[b + k],
+ * pretending that a[b - 1] is minus infinity and a[b + n] is infinity.
+ * In other words, key belongs at index b + k; or in other words,
+ * the first k elements of a should precede key, and the last n - k
+ * should follow it.
+ */
+ private static int gallopLeft(int key, int[] a, int base, int len, int hint,
+ ExposedComparators.OrdinalComparator c) {
+ assert len > 0 && hint >= 0 && hint < len;
+ int lastOfs = 0;
+ int ofs = 1;
+ if (c.compare(key, a[base + hint]) > 0) {
+ // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs]
+ int maxOfs = len - hint;
+ while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) > 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to base
+ lastOfs += hint;
+ ofs += hint;
+ } else { // key <= a[base + hint]
+ // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs]
+ final int maxOfs = hint + 1;
+ while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) <= 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to base
+ int tmp = lastOfs;
+ lastOfs = hint - ofs;
+ ofs = hint - tmp;
+ }
+ assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
+
+ /*
+ * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere
+ * to the right of lastOfs but no farther right than ofs. Do a binary
+ * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs].
+ */
+ lastOfs++;
+ while (lastOfs < ofs) {
+ int m = lastOfs + ((ofs - lastOfs) >>> 1);
+
+ if (c.compare(key, a[base + m]) > 0)
+ lastOfs = m + 1; // a[base + m] < key
+ else
+ ofs = m; // key <= a[base + m]
+ }
+ assert lastOfs == ofs; // so a[base + ofs - 1] < key <= a[base + ofs]
+ return ofs;
+ }
+
+ /**
+ * Like gallopLeft, except that if the range contains an element equal to
+ * key, gallopRight returns the index after the rightmost equal element.
+ *
+ * @param key the key whose insertion point to search for
+ * @param a the array in which to search
+ * @param base the index of the first element in the range
+ * @param len the length of the range; must be > 0
+ * @param hint the index at which to begin the search, 0 <= hint < n.
+ * The closer hint is to the result, the faster this method will run.
+ * @param c the comparator used to order the range, and to search
+ * @return the int k, 0 <= k <= n such that a[b + k - 1] <= key < a[b + k]
+ */
+ private static int gallopRight(int key, int[] a, int base, int len,
+ int hint, ExposedComparators.OrdinalComparator c) {
+ assert len > 0 && hint >= 0 && hint < len;
+
+ int ofs = 1;
+ int lastOfs = 0;
+ if (c.compare(key, a[base + hint]) < 0) {
+ // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs]
+ int maxOfs = hint + 1;
+ while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) < 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to b
+ int tmp = lastOfs;
+ lastOfs = hint - ofs;
+ ofs = hint - tmp;
+ } else { // a[b + hint] <= key
+ // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs]
+ int maxOfs = len - hint;
+ while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) >= 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to b
+ lastOfs += hint;
+ ofs += hint;
+ }
+ assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
+
+ /*
+ * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to
+ * the right of lastOfs but no farther right than ofs. Do a binary
+ * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs].
+ */
+ lastOfs++;
+ while (lastOfs < ofs) {
+ int m = lastOfs + ((ofs - lastOfs) >>> 1);
+
+ if (c.compare(key, a[base + m]) < 0)
+ ofs = m; // key < a[b + m]
+ else
+ lastOfs = m + 1; // a[b + m] <= key
+ }
+ assert lastOfs == ofs; // so a[b + ofs - 1] <= key < a[b + ofs]
+ return ofs;
+ }
+
+ /**
+ * Merges two adjacent runs in place, in a stable fashion. The first
+ * element of the first run must be greater than the first element of the
+ * second run (a[base1] > a[base2]), and the last element of the first run
+ * (a[base1 + len1-1]) must be greater than all elements of the second run.
+ *
+ * For performance, this method should be called only when len1 <= len2;
+ * its twin, mergeHi should be called if len1 >= len2. (Either method
+ * may be called if len1 == len2.)
+ *
+ * @param base1 index of first element in first run to be merged
+ * @param len1 length of first run to be merged (must be > 0)
+ * @param base2 index of first element in second run to be merged
+ * (must be aBase + aLen)
+ * @param len2 length of second run to be merged (must be > 0)
+ */
+ private void mergeLo(int base1, int len1, int base2, int len2) {
+ assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
+
+ // Copy first run into temp array
+ int[] a = this.a; // For performance
+ int[] tmp = ensureCapacity(len1);
+ System.arraycopy(a, base1, tmp, 0, len1);
+
+ int cursor1 = 0; // Indexes into tmp array
+ int cursor2 = base2; // Indexes int a
+ int dest = base1; // Indexes int a
+
+ // Move first element of second run and deal with degenerate cases
+ a[dest++] = a[cursor2++];
+ if (--len2 == 0) {
+ System.arraycopy(tmp, cursor1, a, dest, len1);
+ return;
+ }
+ if (len1 == 1) {
+ System.arraycopy(a, cursor2, a, dest, len2);
+ a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge
+ return;
+ }
+
+ ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance
+ int minGallop = this.minGallop; // " " " " "
+ outer:
+ while (true) {
+ int count1 = 0; // Number of times in a row that first run won
+ int count2 = 0; // Number of times in a row that second run won
+
+ /*
+ * Do the straightforward thing until (if ever) one run starts
+ * winning consistently.
+ */
+ do {
+ assert len1 > 1 && len2 > 0;
+ if (c.compare(a[cursor2], tmp[cursor1]) < 0) {
+ a[dest++] = a[cursor2++];
+ count2++;
+ count1 = 0;
+ if (--len2 == 0)
+ break outer;
+ } else {
+ a[dest++] = tmp[cursor1++];
+ count1++;
+ count2 = 0;
+ if (--len1 == 1)
+ break outer;
+ }
+ } while ((count1 | count2) < minGallop);
+
+ /*
+ * One run is winning so consistently that galloping may be a
+ * huge win. So try that, and continue galloping until (if ever)
+ * neither run appears to be winning consistently anymore.
+ */
+ do {
+ assert len1 > 1 && len2 > 0;
+ count1 = gallopRight(a[cursor2], tmp, cursor1, len1, 0, c);
+ if (count1 != 0) {
+ System.arraycopy(tmp, cursor1, a, dest, count1);
+ dest += count1;
+ cursor1 += count1;
+ len1 -= count1;
+ if (len1 <= 1) // len1 == 1 || len1 == 0
+ break outer;
+ }
+ a[dest++] = a[cursor2++];
+ if (--len2 == 0)
+ break outer;
+
+ count2 = gallopLeft(tmp[cursor1], a, cursor2, len2, 0, c);
+ if (count2 != 0) {
+ System.arraycopy(a, cursor2, a, dest, count2);
+ dest += count2;
+ cursor2 += count2;
+ len2 -= count2;
+ if (len2 == 0)
+ break outer;
+ }
+ a[dest++] = tmp[cursor1++];
+ if (--len1 == 1)
+ break outer;
+ minGallop--;
+ } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
+ if (minGallop < 0)
+ minGallop = 0;
+ minGallop += 2; // Penalize for leaving gallop mode
+ } // End of "outer" loop
+ this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field
+
+ if (len1 == 1) {
+ assert len2 > 0;
+ System.arraycopy(a, cursor2, a, dest, len2);
+ a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge
+ } else if (len1 == 0) {
+ throw new IllegalArgumentException(
+ "Comparison method violates its general contract!");
+ } else {
+ assert len2 == 0;
+ assert len1 > 1;
+ System.arraycopy(tmp, cursor1, a, dest, len1);
+ }
+ }
+
+ /**
+ * Like mergeLo, except that this method should be called only if
+ * len1 >= len2; mergeLo should be called if len1 <= len2. (Either method
+ * may be called if len1 == len2.)
+ *
+ * @param base1 index of first element in first run to be merged
+ * @param len1 length of first run to be merged (must be > 0)
+ * @param base2 index of first element in second run to be merged
+ * (must be aBase + aLen)
+ * @param len2 length of second run to be merged (must be > 0)
+ */
+ private void mergeHi(int base1, int len1, int base2, int len2) {
+ assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
+
+ // Copy second run into temp array
+ int[] a = this.a; // For performance
+ int[] tmp = ensureCapacity(len2);
+ System.arraycopy(a, base2, tmp, 0, len2);
+
+ int cursor1 = base1 + len1 - 1; // Indexes into a
+ int cursor2 = len2 - 1; // Indexes into tmp array
+ int dest = base2 + len2 - 1; // Indexes into a
+
+ // Move last element of first run and deal with degenerate cases
+ a[dest--] = a[cursor1--];
+ if (--len1 == 0) {
+ System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2);
+ return;
+ }
+ if (len2 == 1) {
+ dest -= len1;
+ cursor1 -= len1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, len1);
+ a[dest] = tmp[cursor2];
+ return;
+ }
+
+ ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance
+ int minGallop = this.minGallop; // " " " " "
+ outer:
+ while (true) {
+ int count1 = 0; // Number of times in a row that first run won
+ int count2 = 0; // Number of times in a row that second run won
+
+ /*
+ * Do the straightforward thing until (if ever) one run
+ * appears to win consistently.
+ */
+ do {
+ assert len1 > 0 && len2 > 1;
+ if (c.compare(tmp[cursor2], a[cursor1]) < 0) {
+ a[dest--] = a[cursor1--];
+ count1++;
+ count2 = 0;
+ if (--len1 == 0)
+ break outer;
+ } else {
+ a[dest--] = tmp[cursor2--];
+ count2++;
+ count1 = 0;
+ if (--len2 == 1)
+ break outer;
+ }
+ } while ((count1 | count2) < minGallop);
+
+ /*
+ * One run is winning so consistently that galloping may be a
+ * huge win. So try that, and continue galloping until (if ever)
+ * neither run appears to be winning consistently anymore.
+ */
+ do {
+ assert len1 > 0 && len2 > 1;
+ count1 = len1 - gallopRight(tmp[cursor2], a, base1, len1, len1 - 1, c);
+ if (count1 != 0) {
+ dest -= count1;
+ cursor1 -= count1;
+ len1 -= count1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, count1);
+ if (len1 == 0)
+ break outer;
+ }
+ a[dest--] = tmp[cursor2--];
+ if (--len2 == 1)
+ break outer;
+
+ count2 = len2 - gallopLeft(a[cursor1], tmp, 0, len2, len2 - 1, c);
+ if (count2 != 0) {
+ dest -= count2;
+ cursor2 -= count2;
+ len2 -= count2;
+ System.arraycopy(tmp, cursor2 + 1, a, dest + 1, count2);
+ if (len2 <= 1) // len2 == 1 || len2 == 0
+ break outer;
+ }
+ a[dest--] = a[cursor1--];
+ if (--len1 == 0)
+ break outer;
+ minGallop--;
+ } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
+ if (minGallop < 0)
+ minGallop = 0;
+ minGallop += 2; // Penalize for leaving gallop mode
+ } // End of "outer" loop
+ this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field
+
+ if (len2 == 1) {
+ assert len1 > 0;
+ dest -= len1;
+ cursor1 -= len1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, len1);
+ a[dest] = tmp[cursor2]; // Move first elt of run2 to front of merge
+ } else if (len2 == 0) {
+ throw new IllegalArgumentException(
+ "Comparison method violates its general contract!");
+ } else {
+ assert len1 == 0;
+ assert len2 > 0;
+ System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2);
+ }
+ }
+
+ /**
+ * Ensures that the external array tmp has at least the specified
+ * number of elements, increasing its size if necessary. The size
+ * increases exponentially to ensure amortized linear time complexity.
+ *
+ * @param minCapacity the minimum required capacity of the tmp array
+ * @return tmp, whether or not it grew
+ */
+ private int[] ensureCapacity(int minCapacity) {
+ if (tmp.length < minCapacity) {
+ // Compute smallest power of 2 > minCapacity
+ int newSize = minCapacity;
+ newSize |= newSize >> 1;
+ newSize |= newSize >> 2;
+ newSize |= newSize >> 4;
+ newSize |= newSize >> 8;
+ newSize |= newSize >> 16;
+ newSize++;
+
+ if (newSize < 0) // Not bloody likely!
+ newSize = minCapacity;
+ else
+ newSize = Math.min(newSize, a.length >>> 1);
+
+ tmp = new int[newSize];
+ }
+ return tmp;
+ }
+
+ /**
+ * Checks that fromIndex and toIndex are in range, and throws an
+ * appropriate exception if they aren't.
+ *
+ * @param arrayLen the length of the array
+ * @param fromIndex the index of the first element of the range
+ * @param toIndex the index after the last element of the range
+ * @throws IllegalArgumentException if fromIndex > toIndex
+ * @throws ArrayIndexOutOfBoundsException if fromIndex < 0
+ * or toIndex > arrayLen
+ */
+ private static void rangeCheck(int arrayLen, int fromIndex, int toIndex) {
+ if (fromIndex > toIndex)
+ throw new IllegalArgumentException("fromIndex(" + fromIndex +
+ ") > toIndex(" + toIndex+")");
+ if (fromIndex < 0)
+ throw new ArrayIndexOutOfBoundsException(fromIndex);
+ if (toIndex > arrayLen)
+ throw new ArrayIndexOutOfBoundsException(toIndex);
+ }
+}
\ No newline at end of file
Index: lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java Fri Sep 17 21:57:54 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java Fri Sep 17 21:57:54 CEST 2010
@@ -0,0 +1,514 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.*;
+
+
+public class TestExposedCache extends LuceneTestCase {
+ public static final int DOCCOUNT = 10;
+ private ExposedHelper helper;
+ private ExposedCache cache;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ cache = new ExposedCache(FieldCache.DEFAULT);
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ cache.purgeAllCaches();
+ //helper.close();
+ }
+
+ public void testPlainSortDump() throws Exception {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN, new MockAnalyzer());
+ Query q = qp.parse("true");
+ Sort aSort = new Sort(new SortField("a", new Locale("da")));
+
+ TopFieldDocs docs = searcher.search(q, null, 100, aSort);
+ dumpDocs(reader, docs, "a");
+ reader.close();
+ }
+
+ public void testIndirectIndex() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+
+ PackedInts.Reader orderedOrdinals = provider.getOrderedOrdinals();
+ Iterator iterator = provider.getIterator(false);
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+ assertEquals("The provided term should match ordinal lookup term",
+ tuple.term.utf8ToString(),
+ provider.getTerm(tuple.ordinal).utf8ToString());
+ assertEquals("The provided term should match indirect lookup term",
+ tuple.term.utf8ToString(),
+ provider.getOrderedTerm(tuple.indirect).utf8ToString());
+ }
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ String exposed = provider.getOrderedTerm(
+ provider.getDocToSingleIndirect().get(docID)).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }
+ }
+
+ public void testIndirectSegment() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0]; // SegmentReader
+
+ ExposedRequest.Field fRequest = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, fRequest, true, true);
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ long indirect = provider.getDocToSingleIndirect().get(docID);
+ String exposed = indirect == -1 ? null :
+ provider.getOrderedTerm(indirect).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }
+ }
+
+ public void testDocCount() throws IOException {
+ int docs = 12000;
+ helper.createIndex(docs, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ ExposedCache.getInstance().purgeAllCaches();
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ assertEquals("The number of documents should be correct",
+ reader.maxDoc(), provider.getMaxDoc());
+ }
+
+ public void testReopen() throws IOException, ParseException {
+ final int RUNS = 5;
+ final int UPDATE_SIZE = 5000;
+ List exposedSortTimes = new ArrayList(RUNS);
+
+ helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 2);
+
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer());
+ Query query = qp.parse(ExposedHelper.ALL);
+ Sort plainSort = new Sort(new SortField("a", new Locale("da")));
+
+ exposedSortTimes.add(testReopen(reader, query, plainSort));
+ for (int i = 1 ; i < RUNS ; i++) {
+ System.out.println("\nModifying index and re-opening reader "
+ + (i+1) + "/" + RUNS);
+ helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 1);
+ IndexReader oldReader = reader;
+ reader = reader.reopen();
+ assertFalse("The index should change", oldReader == reader);
+ oldReader.close();
+ exposedSortTimes.add(testReopen(reader, query, plainSort));
+ }
+ reader.close();
+ long total = 0;
+ for (long t: exposedSortTimes) {
+ total += t;
+ }
+ System.out.println(
+ "\n" + RUNS + " exposed sortings with doc#-delta " + UPDATE_SIZE
+ + " between each took " + total
+ + " ms split in " + exposedSortTimes);
+ }
+
+ private long testReopen(
+ IndexReader reader, Query query, Sort plainSort) throws IOException {
+ final int docCount = reader.maxDoc();
+ final int MAX_HITS = 100;
+ Sort exposedSort = new Sort(new SortField(
+ "a",
+ new ExposedFieldComparatorSource(reader, new Locale("da"))));
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+ long plainTime = System.currentTimeMillis();
+ TopFieldDocs plainDocs = searcher.search(
+ query.createWeight(searcher), null, MAX_HITS, plainSort, true);
+ plainTime = System.currentTimeMillis() - plainTime;
+
+ long exposedTime = System.currentTimeMillis();
+ TopFieldDocs exposedDocs = searcher.search(
+ query.createWeight(searcher), null, MAX_HITS, exposedSort, true);
+ exposedTime = System.currentTimeMillis() - exposedTime;
+ System.out.println(
+ "Reopen sort test extracted "
+ + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits in "
+ + plainTime + " ms for standard Lucene collator sort and "
+ + exposedTime + " ms for exposed sort");
+// dumpDocs(reader, plainDocs, sortField);
+// dumpDocs(reader, exposedDocs, sortField);
+ assertEquals("The two search results should be equal",
+ plainDocs, exposedDocs);
+ return exposedTime;
+ }
+
+ public void testExposedSortWithNullValues() throws Exception {
+ testExposedSort(ExposedHelper.EVEN_NULL, DOCCOUNT, false);
+ }
+
+ public void testExposedSortWithRandom() throws Exception {
+ testExposedSort("a", DOCCOUNT, false);
+ }
+
+ public void testSpeedExposedSortWithRandom() throws Exception {
+// testExposedSort("a", 150000, true); Fails at GrowingMutable.get from compareBottom with java.lang.ArrayIndexOutOfBoundsException: 22674
+ testExposedSort("a", 20000, true);
+ }
+
+ private void testExposedSort(String sortField, int docCount, boolean feedback)
+ throws IOException, ParseException {
+ helper.createIndex(docCount, Arrays.asList("a", "b"), 20, 2);
+ testExposedSort(ExposedHelper.INDEX_LOCATION, new Locale("da"),
+ sortField, docCount, feedback);
+ }
+
+ public void testExternallyGeneratedIndex() throws Exception {
+ File external = new File("/home/te/projects/lucene4index");
+ if (!external.exists()) {
+ return;
+ }
+ testExposedSort(external, new Locale("da"), "sort_title", 10, true);
+ }
+
+ public void testExposedNaturalOrder() throws IOException, ParseException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a"), 20, 2);
+ testExposedSort(ExposedHelper.INDEX_LOCATION, null, "a", 10, true);
+ }
+
+ // locale can be null
+ private void testExposedSort(File index, Locale locale, String sortField,
+ int docCount, boolean feedback)
+ throws IOException, ParseException {
+ final int MAX_HITS = 50;
+ int retries = feedback ? 5 : 1;
+
+ IndexReader reader = IndexReader.open(FSDirectory.open(index), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer());
+ Query q = qp.parse(ExposedHelper.ALL);
+ Sort aPlainSort = locale == null ?
+ new Sort(new SortField(sortField, SortField.STRING)) :
+ new Sort(new SortField(sortField, locale));
+ Sort aExposedSort = new Sort(new SortField(
+ sortField, new ExposedFieldComparatorSource(reader, locale)));
+
+ for (int run = 0 ; run < retries ; run++) {
+ long plainTime = System.currentTimeMillis();
+ TopFieldDocs plainDocs = searcher.search(
+ q.createWeight(searcher), null, MAX_HITS, aPlainSort, true);
+ plainTime = System.currentTimeMillis() - plainTime;
+
+ long exposedTime = System.currentTimeMillis();
+ TopFieldDocs exposedDocs = searcher.search(
+ q.createWeight(searcher), null, MAX_HITS, aExposedSort, true);
+ exposedTime = System.currentTimeMillis() - exposedTime;
+ System.out.println(
+ "Sorting on field " + sortField + " with "
+ + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits took "
+ + plainTime + " ms for standard Lucene collator sort and "
+ + exposedTime + " ms for exposed sort");
+// dumpDocs(reader, plainDocs, sortField);
+// dumpDocs(reader, exposedDocs, sortField);
+ assertEquals("The two search results should be equal",
+ plainDocs, exposedDocs);
+ }
+ reader.close();
+ }
+
+ public void testRawSort() throws IOException {
+// final int[] SIZES = new int[]{10, 1000, 20000, 100000};
+ final int[] SIZES = new int[]{20000};
+ final int RUNS = 3;
+ Random random = new Random(87);
+ long createTime = System.currentTimeMillis();
+ for (int size: SIZES) {
+ List terms = new ArrayList(size);
+ for (int termPos = 0 ; termPos < size ; termPos++) {
+ String term = ExposedHelper.getRandomString(
+ random, ExposedHelper.CHARS, 1, 20) + termPos;
+ terms.add(new BytesRef(term));
+ }
+ System.out.println("Created " + size + " random BytesRefs in "
+ + (System.currentTimeMillis() - createTime) + "ms");
+
+ Comparator comparator = ExposedComparators.collatorToBytesRef(
+ Collator.getInstance(new Locale("da")));
+
+ for (int run = 0 ; run < RUNS ; run++) {
+ Collections.shuffle(terms);
+
+ long sortTime = System.currentTimeMillis();
+ Collections.sort(terms, comparator);
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println("Dumb sorted "
+ + ExposedUtil.time("bytesrefs", terms.size(), sortTime));
+
+ long keyTime = System.currentTimeMillis();
+ Collator plainCollator = Collator.getInstance(new Locale("da"));
+ List keys = new ArrayList(size);
+ for (BytesRef term: terms) {
+ keys.add(plainCollator.getCollationKey(term.utf8ToString()));
+ }
+ keyTime = System.currentTimeMillis() - keyTime;
+ System.out.println("Created "
+ + ExposedUtil.time("CollatorKeys", size, keyTime));
+ Collections.shuffle(keys);
+
+ long keySortTime = System.currentTimeMillis();
+ Collections.sort(keys);
+ keySortTime = System.currentTimeMillis() - keySortTime;
+ System.out.println("Collator sorted "
+ + ExposedUtil.time("CollatorKeys", size, keySortTime));
+ }
+ System.out.println("");
+ }
+ }
+
+ public void testPlainOrdinalAccess() throws IOException {
+ final int[] SIZES = new int[]{10, 1000, 50000};
+ final int RETRIES = 6;
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0];
+ TermsEnum terms = reader.fields().terms("a").iterator();
+ long num = reader.fields().terms("a").getUniqueTermCount();
+
+ Random random = new Random();
+ for (int i = 0 ; i < RETRIES ; i++) {
+ boolean doRandom = i % 2 == 0;
+ long firstHalf = System.currentTimeMillis();
+ for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) {
+ terms.seek(doRandom ? random.nextInt(ordinal+1) : ordinal);
+ BytesRef term = terms.term();
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ firstHalf = System.currentTimeMillis() - firstHalf;
+
+ long secondHalf = System.currentTimeMillis();
+ for (long ordinal = num / 2 ; ordinal < num ; ordinal++) {
+ terms.seek(doRandom ? random.nextInt((int) (ordinal+1)) : ordinal);
+ BytesRef term = terms.term();
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ secondHalf = System.currentTimeMillis() - secondHalf;
+
+ System.out.println("Seeked " + num + " ordinals "
+ + (doRandom ? "randomly" : "sequent.") + ". " +
+ "First half: " + firstHalf + " ms, second half: "
+ + secondHalf + " ms"
+ + (firstHalf + secondHalf == 0 ? "" : ". "
+ + num / (firstHalf + secondHalf) + " seeks/ms"));
+ }
+ reader.close();
+ }
+ }
+
+ public void testExposedOrdinalAccess() throws IOException {
+ final int[] SIZES = new int[]{10, 1000, 50000};
+ final int RETRIES = 6;
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents by exposed");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0];
+ ExposedRequest.Field request = new ExposedRequest.Field("a", null, "foo");
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, request, true, true);
+ long num = provider.getMaxDoc();
+
+ Random random = new Random();
+ for (int i = 0 ; i < RETRIES ; i++) {
+ boolean doRandom = i % 2 == 0;
+ long firstHalf = System.currentTimeMillis();
+ for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) {
+ BytesRef term = provider.getTerm(
+ doRandom ? random.nextInt(ordinal+1) : ordinal);
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ firstHalf = System.currentTimeMillis() - firstHalf;
+
+ long secondHalf = System.currentTimeMillis();
+ for (long ordinal = num / 2 ; ordinal < num ; ordinal++) {
+ BytesRef term = provider.getTerm(
+ doRandom ? random.nextInt((int) (ordinal+1)) : ordinal);
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ secondHalf = System.currentTimeMillis() - secondHalf;
+
+ System.out.println("Requested " + (num-1) + " ordinals "
+ + (doRandom ? "randomly" : "sequent.") + ". " +
+ "First half: " + firstHalf + " ms, second half: "
+ + secondHalf + " ms"
+ + (firstHalf + secondHalf == 0 ? "" : ". "
+ + num / (firstHalf + secondHalf) + " seeks/ms"));
+ }
+ reader.close();
+ }
+ }
+ /*
+
+Measuring for 50000 documents
+Created 50000 document index with 4 fields with average term length 10 and total size 7MB in 2409ms
+org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@38a36b53
+org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@d694eca
+Chunk sorted 0-19999: 20000 ordinals in 1061 ms: ~= 18 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/255151, lookups=20000 (66 ms ~= 300 lookups/ms), readAheads=19999
+Chunk sorted 20000-25000: 5001 ordinals in 179 ms: ~= 27 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/54131, lookups=5001 (2 ms ~= 1722 lookups/ms), readAheads=5000
+Chunk merged 2 chunks in 234 ms: ~= 0 chunks/ms aka 25001 terms in 234 ms: ~= 106 terms/ms: CachedTermProvider cacheSize=20000, misses=15/49938, lookups=25804 (32 ms ~= 789 lookups/ms), readAheads=25797
+Chunk total sort for field a: 25001 terms in 1478 ms: ~= 16 terms/ms
+Chunk sorted 0-19999: 20000 ordinals in 523 ms: ~= 38 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/257147, lookups=20000 (12 ms ~= 1566 lookups/ms), readAheads=19999
+Chunk sorted 20000-24998: 4999 ordinals in 112 ms: ~= 44 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/53847, lookups=4999 (2 ms ~= 1752 lookups/ms), readAheads=4998
+Chunk merged 2 chunks in 116 ms: ~= 0 chunks/ms aka 24999 terms in 116 ms: ~= 215 terms/ms: CachedTermProvider cacheSize=20000, misses=21/49996, lookups=25217 (15 ms ~= 1674 lookups/ms), readAheads=25210
+Chunk total sort for field a: 24999 terms in 753 ms: ~= 33 terms/ms
+FacetGroup total iterator construction: 50000 ordinals in 2246 ms: ~= 22 ordinals/ms
+TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=411/25001, lookups=26406 (33 ms ~= 798 lookups/ms), readAheads=26181
+TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=390/24999, lookups=26092 (32 ms ~= 799 lookups/ms), readAheads=25868
+FacetGroup ordinal iterator depletion from 2 providers: 50001 ordinals in 426 ms: ~= 117 ordinals/ms
+Got 50001 ordered ordinals in 2673ms: ~18 terms/ms
+
+ */
+ public void testTiming() throws IOException {
+// final int[] SIZES = new int[]{10, 1000, 20000, 50000};
+ final int[] SIZES = new int[]{20000};
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ ExposedCache.getInstance().purgeAllCaches();
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ assertEquals("The number of ordinal accessible terms should match",
+ size, provider.getOrdinalTermCount());
+
+ long sortTime = System.currentTimeMillis();
+ PackedInts.Reader orderedOrdinals = provider.getOrderedOrdinals();
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println("Got " + orderedOrdinals.size()
+ + " ordered ordinals in " + sortTime + "ms"
+ + (sortTime == 0 ? "" : ": ~" + (orderedOrdinals.size() / sortTime)
+ + " terms/ms"));
+ reader.close();
+/* Iterator iterator = provider.getIterator(false);
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+ assertEquals("The provided term should match ordinal lookup term",
+ tuple.term.utf8ToString(),
+ provider.getTerm(tuple.ordinal).utf8ToString());
+ assertEquals("The provided term should match indirect lookup term",
+ tuple.term.utf8ToString(),
+ provider.getOrderedTerm(tuple.indirect).utf8ToString());
+ }
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ String exposed = provider.getOrderedTerm(
+ provider.getDocToSingleIndirect().get(docID)).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }*/
+ }
+ }
+
+ public static void assertEquals(String message, TopFieldDocs expected,
+ TopFieldDocs actual) {
+ assertEquals(message + ". Expected length " + expected.scoreDocs.length
+ + " got " + actual.scoreDocs.length,
+ expected.scoreDocs.length, actual.scoreDocs.length);
+ for (int i = 0 ; i < actual.scoreDocs.length ; i++) {
+ ScoreDoc e = expected.scoreDocs[i];
+ ScoreDoc a = actual.scoreDocs[i];
+ assertEquals(message + ". The docID for hit#" + (i+1)
+ + "/" + expected.scoreDocs.length + " should be correct",
+ e.doc, a.doc);
+
+ String ef = ((FieldDoc)e).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)e).fields[0]).utf8ToString();
+ String af = ((FieldDoc)a).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)a).fields[0]).utf8ToString();
+ assertEquals(message + ". The sort value for hit#" + (i+1)
+ + "/" + expected.scoreDocs.length + " should be "
+ + ef + " but was " + af,
+ ef, af);
+ }
+ }
+
+ private void dumpDocs(IndexReader reader, TopFieldDocs docs, String field)
+ throws IOException {
+ final int MAX = 20;
+ int num = Math.min(docs.scoreDocs.length, MAX);
+ System.out.println("Dumping " + num +"/" + docs.scoreDocs.length + " hits");
+ int count = 0;
+ for (ScoreDoc doc: docs.scoreDocs) {
+ if (count++ > num) {
+ break;
+ }
+ String sortTerm = ((FieldDoc)doc).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)doc).fields[0]).utf8ToString();
+ System.out.println(
+ doc.doc + " ID=" + reader.document(doc.doc).get(ExposedHelper.ID)
+ + ", field " + field + "=" + reader.document(doc.doc).get(field)
+ + ", sort term=" + sortTerm);
+ }
+ }
+
+
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/facet/TestExposedFacets.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,559 @@
+package org.apache.lucene.search.exposed.facet;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+import junit.framework.TestCase;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.exposed.ExposedCache;
+import org.apache.lucene.search.exposed.ExposedFieldComparatorSource;
+import org.apache.lucene.search.exposed.ExposedHelper;
+import org.apache.lucene.search.exposed.ExposedSettings;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+
+import javax.xml.stream.XMLStreamException;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+public class TestExposedFacets extends TestCase {
+ private ExposedHelper helper;
+ private ExposedCache cache;
+
+ public TestExposedFacets(String name) {
+ super(name);
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ cache = new ExposedCache(FieldCache.DEFAULT);
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ cache.purgeAllCaches();
+ }
+
+
+ public static Test suite() {
+ return new TestSuite(TestExposedFacets.class);
+ }
+
+ public static final String SIMPLE_REQUEST =
+ "\n" +
+ "\n" +
+ " even:true\n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+// " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ "";
+ public void testSimpleFacet() throws Exception {
+ final int DOCCOUNT = 1000; // Try with 5
+ final int TERM_LENGTH = 20;
+ final int MIN_SEGMENTS = 2;
+ final List FIELDS = Arrays.asList("a", "b");
+
+ helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ Query q = qp.parse("true");
+ searcher.search(q, TopScoreDocCollector.create(10, false));
+ long preMem = getMem();
+ long facetStructureTime = System.currentTimeMillis();
+
+ CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2);
+
+ FacetRequest request = FacetRequest.parseXML(SIMPLE_REQUEST);
+ CollectorPool collectorPool = poolFactory.acquire(reader, request);
+ facetStructureTime = System.currentTimeMillis() - facetStructureTime;
+
+ TagCollector collector;
+ FacetResponse response = null;
+ String sQuery = request.getQuery();
+ for (int i = 0 ; i < 5 ; i++) {
+ collector = collectorPool.acquire(sQuery);
+ long countStart = System.currentTimeMillis();
+ if (collector.getQuery() == null) { // Fresh collector
+ searcher.search(q, collector);
+// collector.collectAllValid(reader);
+ long countTime = System.currentTimeMillis() - countStart;
+ collector.setCountTime(countTime);
+ }
+ response = collector.extractResult(request);
+ if (collector.getQuery() != null) { // Cached count
+ response.setCountingCached(true);
+ }
+ long totalTime = System.currentTimeMillis() - countStart;
+ response.setTotalTime(totalTime);
+ System.out.println("Collection #" + i + " for " + DOCCOUNT
+ + " documents in "
+ + getTime(System.currentTimeMillis()-countStart));
+ collectorPool.release(sQuery, collector);
+ }
+ System.out.println("Document count = " + DOCCOUNT);
+ System.out.println("Facet startup time = " + getTime(facetStructureTime));
+ System.out.println("Mem usage: preFacet=" + preMem
+ + " MB, postFacet=" + getMem() + " MB");
+ if (response != null) {
+ System.out.println(response.toXML());
+ }
+ }
+
+ public static final String LOOKUP_REQUEST =
+ "\n" +
+ "\n" +
+ " even:true\n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ "";
+ public void testIndexLookup() throws Exception {
+ final int DOCCOUNT = 20000;
+ final int TERM_LENGTH = 20;
+ final int MIN_SEGMENTS = 2;
+ final List FIELDS = Arrays.asList("a");
+ helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ Query q = qp.parse("true");
+ searcher.search(q, TopScoreDocCollector.create(10, false));
+ System.out.println(
+ "Index opened and test-sort performed, commencing faceting...");
+ long preMem = getMem();
+
+ CollectorPoolFactory poolFactory = new CollectorPoolFactory(2, 4, 2);
+
+ FacetResponse response = null;
+ assertTrue("There must be at least 10 documents in the index",
+ DOCCOUNT >= 10);
+ final int span = DOCCOUNT / 10;
+ long firstTime = -1;
+ long subsequents = 0;
+ int subCount = 0;
+ long poolTime = -1;
+ for (char prefix = 'A' ; prefix < 'X' ; prefix++) {
+ for (int offset = -span ; offset < span ; offset += span / 2) {
+ FacetRequest request = FacetRequest.parseXML(
+ LOOKUP_REQUEST.replace("myprefix", "a_" + prefix).
+ replace("myoffset", Integer.toString(offset)));
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis();
+ }
+ CollectorPool collectorPool = poolFactory.acquire(reader, request);
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis() - poolTime;
+ System.out.println("Facet structure build finished in "
+ + getTime(poolTime) + ". Running requests...\n");
+ }
+
+ long countStart = System.currentTimeMillis();
+ String sQuery = request.getQuery();
+ TagCollector collector = collectorPool.acquire(sQuery);
+ if (collector.getQuery() == null) { // Fresh collector
+ searcher.search(q, collector);
+ long countTime = System.currentTimeMillis() - countStart;
+ collector.setCountTime(countTime);
+ }
+ response = collector.extractResult(request);
+ assertNotNull("Extracting XML response should work", response.toXML());
+ if (collector.getQuery() != null) { // Cached count
+ response.setCountingCached(true);
+ }
+ long totalTime = System.currentTimeMillis() - countStart;
+ if (firstTime == -1) {
+ firstTime = totalTime;
+ } else {
+ subCount++;
+ subsequents += totalTime;
+ }
+ response.setTotalTime(totalTime);
+/* System.out.println("Collection for prefix " + prefix + " and offset "
+ + offset + " for " + DOCCOUNT + " documents took "
+ + (System.currentTimeMillis()-countStart) + "ms");*/
+ collectorPool.release(sQuery, collector);
+ }
+ }
+ System.out.println("Document count = " + DOCCOUNT);
+ System.out.println("Facet structure build time = " + getTime(poolTime));
+ System.out.println("First facet call = " + getTime(firstTime));
+ System.out.println("Average of " + subCount + " subsequent calls = "
+ + getTime(subsequents / subCount));
+ System.out.println("Mem usage: preFacet=" + preMem
+ + " MB, postFacet=" + getMem() + " MB");
+ if (response != null) {
+ System.out.println(response.toXML());
+ }
+
+ }
+
+ public void testBasicSearch() throws IOException, ParseException {
+ final int DOCCOUNT = 200;
+ final int TERM_LENGTH = 20;
+ final int MIN_SEGMENTS = 2;
+ final List FIELDS = Arrays.asList("a");
+ helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS);
+ final File LOCATION = ExposedHelper.INDEX_LOCATION;
+
+ final String field = "facet";
+ final String term = "A";
+
+ assertTrue("No index at " + LOCATION.getAbsolutePath()
+ + ". Please build a test index (you can use one from on of the other" +
+ " JUnit tests in TestExposedFacets) and correct the path",
+ LOCATION.exists());
+
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, field,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ Query q = qp.parse(term);
+ System.out.println(q.toString());
+ String sQuery = field + ":" + term;
+
+ TopDocsCollector collector = TopScoreDocCollector.create(10, false);
+ searcher.search(q, collector);
+ assertTrue("There should be some hits for '" + sQuery + "'",
+ collector.getTotalHits() >0);
+ }
+
+ // MB
+ private long getMem() {
+ System.gc();
+ return (Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory()) / 1048576;
+ }
+
+ public static final String SCALE_REQUEST =
+ "\n" +
+ "\n" +
+ " even:true\n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ "";
+ /*
+ Performs sorted search, faceting and index lookup
+ */
+ public void testScale() throws Exception {
+ long totalTime = -System.currentTimeMillis();
+
+ ExposedSettings.priority = ExposedSettings.PRIORITY.speed;
+
+ File LOCATION = new File("/home/te/projects/index2M");
+ if (!LOCATION.exists()) {
+ final int DOCCOUNT = 20000;
+ final int TERM_LENGTH = 20;
+ final int MIN_SEGMENTS = 2;
+ final List FIELDS = Arrays.asList("a");
+ System.err.println("No index at " + LOCATION + ". A test index with " +
+ DOCCOUNT + " documents will be build");
+ helper.createIndex(DOCCOUNT, FIELDS, TERM_LENGTH, MIN_SEGMENTS);
+ LOCATION = ExposedHelper.INDEX_LOCATION;
+ }
+ assertTrue("No index at " + LOCATION.getAbsolutePath()
+ + ". Please build a test index (you can use one from on of the other" +
+ " JUnit tests in TestExposedFacets) and correct the path",
+ LOCATION.exists());
+
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ Query q = qp.parse("true");
+ String sQuery = "even:true";
+
+ StringWriter sw = new StringWriter(10000);
+ sw.append("Index = " + LOCATION.getAbsolutePath() + " (" + reader.maxDoc()
+ + " documents)\n");
+
+ searcher.search(q, TopScoreDocCollector.create(10, false));
+ sw.append("used heap after loading index and performing a simple search: "
+ + getMem() + " MB\n");
+ sw.append("Maximum possible memory (Runtime.getRuntime().maxMemory()): "
+ + Runtime.getRuntime().maxMemory() / 1048576 + " MB\n");
+ System.out.println("Index opened and test search performed successfully");
+
+ sw.append("\n");
+
+ // Tests
+ CollectorPoolFactory poolFactory = new CollectorPoolFactory(6, 4, 2);
+
+ qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ q = qp.parse("true");
+ sQuery = "even:true";
+ testScale(poolFactory, searcher, q, sQuery, sw);
+
+ qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.MULTI,
+ new MockAnalyzer(MockTokenizer.WHITESPACE, false));
+ q = qp.parse("A");
+ sQuery = "multi:A"; // Strictly it's "facet:A", but that is too confusing
+ testScale(poolFactory, searcher, q, sQuery, sw);
+
+
+ sw.append("\nFree memory with sort, facet and index lookup structures " +
+ "intact: " + getMem() + " MB\n");
+ System.out.println("*************** Finished testing in "
+ + getTime(totalTime + System.currentTimeMillis()) + ". Result follows");
+
+ System.out.println("");
+ System.out.println(sw.toString());
+ }
+
+ private void testScale(CollectorPoolFactory poolFactory,
+ IndexSearcher searcher, Query query, String sQuery, StringWriter result)
+ throws IOException, XMLStreamException {
+ System.out.println("- Testing sorted search for " + sQuery);
+ testSortedSearch(searcher, query, sQuery, null, result);
+ result.append("\n");
+
+ // Faceting
+ System.out.println("- Testing faceting for " + sQuery);
+ TestFaceting(poolFactory, searcher, query, sQuery, result);
+
+ // Index lookup
+ System.out.println("- Testing index lookup for " + sQuery);
+ TestIndexLookup(poolFactory, searcher, query, sQuery, result);
+
+ }
+
+ private void TestFaceting(
+ CollectorPoolFactory poolFactory, IndexSearcher searcher, Query q,
+ String sQuery, StringWriter result)
+ throws XMLStreamException, IOException {
+ IndexReader reader = searcher.getIndexReader();
+ long facetStructureTime = -System.currentTimeMillis();
+
+ FacetResponse response = null;
+ int DOCCOUNT = reader.maxDoc();
+ assertTrue("There must be at least 10 documents in the index",
+ DOCCOUNT >= 10);
+ final int span = DOCCOUNT / 10;
+ long firstTime = -1;
+ long subsequents = 0;
+ int subCount = 0;
+ long poolTime = -1;
+ for (int i = 0 ; i < 5 ; i++) {
+ FacetRequest request = FacetRequest.parseXML(SCALE_REQUEST);
+ request.setQuery(sQuery);
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis();
+ }
+ CollectorPool collectorPool = poolFactory.acquire(reader, request);
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis() - poolTime;
+ result.append("Facet pool acquisition for " +
+ "for \"" + sQuery + "\" with structure " + request.getGroupKey()
+ + ": " + getTime(poolTime) + "\n");
+ }
+
+ long countStart = System.currentTimeMillis();
+ TagCollector collector = collectorPool.acquire(null); // No caching
+ if (collector.getQuery() == null) { // Fresh collector
+ searcher.search(q, collector);
+ long countTime = System.currentTimeMillis() - countStart;
+ collector.setCountTime(countTime);
+ }
+ response = collector.extractResult(request);
+ assertNotNull("Extracting XML response should work", response.toXML());
+ if (collector.getQuery() != null) { // Cached count
+ response.setCountingCached(true);
+ }
+ long totalTime = System.currentTimeMillis() - countStart;
+ if (firstTime == -1) {
+ firstTime = totalTime;
+ result.append("First faceting for " + sQuery + ": "
+ + getTime(totalTime) + "\n");
+ } else {
+ subCount++;
+ subsequents += totalTime;
+ }
+ response.setTotalTime(totalTime);
+/* System.out.println("Collection for prefix " + prefix + " and offset "
+ + offset + " for " + DOCCOUNT + " documents took "
+ + (System.currentTimeMillis()-countStart) + "ms");*/
+ collectorPool.release(null, collector); // No caching
+ }
+ result.append("Subsequent " + subCount + " faceting calls (count caching " +
+ "disabled) response times: " + getTime(subsequents / subCount) + "\n");
+ assertNotNull("There should be a response", response);
+ result.append(response.toXML()).append("\n");
+ }
+
+ private void TestIndexLookup(
+ CollectorPoolFactory poolFactory, IndexSearcher searcher, Query q,
+ String sQuery, StringWriter result)
+ throws XMLStreamException, IOException {
+ IndexReader reader = searcher.getIndexReader();
+ long facetStructureTime = -System.currentTimeMillis();
+
+ FacetResponse response = null;
+ int DOCCOUNT = reader.maxDoc();
+ assertTrue("There must be at least 10 documents in the index",
+ DOCCOUNT >= 10);
+ final int span = DOCCOUNT / 10;
+ long firstTime = -1;
+ long subsequents = 0;
+ int subCount = 0;
+ long poolTime = -1;
+ for (char prefix = 'A' ; prefix < 'X' ; prefix++) {
+ for (int offset = -span ; offset < span ; offset += span / 2) {
+ FacetRequest request = FacetRequest.parseXML(
+ LOOKUP_REQUEST.replace("myprefix", "a_" + prefix).
+ replace("myoffset", Integer.toString(offset)));
+ request.setQuery(sQuery);
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis();
+ }
+ CollectorPool collectorPool = poolFactory.acquire(reader, request);
+ if (firstTime == -1) {
+ poolTime = System.currentTimeMillis() - poolTime;
+ result.append("Initial lookup pool request (might result in structure" +
+ " building): " + getTime(poolTime) + "\n");
+ }
+
+ long countStart = System.currentTimeMillis();
+ TagCollector collector = collectorPool.acquire(sQuery);
+ if (collector.getQuery() == null) { // Fresh collector
+ searcher.search(q, collector);
+ long countTime = System.currentTimeMillis() - countStart;
+ collector.setCountTime(countTime);
+ }
+ response = collector.extractResult(request);
+ assertNotNull("Extracting XML response should work", response.toXML());
+ if (collector.getQuery() != null) { // Cached count
+ response.setCountingCached(true);
+ }
+ long totalTime = System.currentTimeMillis() - countStart;
+ if (firstTime == -1) {
+ firstTime = totalTime;
+ result.append("First index lookup for \"" + sQuery + "\": "
+ + getTime(firstTime) + "\n");
+ } else {
+ subCount++;
+ subsequents += totalTime;
+ }
+ response.setTotalTime(totalTime);
+/* System.out.println("Collection for prefix " + prefix + " and offset "
+ + offset + " for " + DOCCOUNT + " documents took "
+ + (System.currentTimeMillis()-countStart) + "ms");*/
+ collectorPool.release(sQuery, collector);
+ }
+ }
+ result.append("Subsequent " + subCount + " index lookups average " +
+ "response times: " + getTime(subsequents / subCount) + "\n");
+ result.append(response.toXML()).append("\n");
+ }
+
+ private void testSortedSearch(IndexSearcher searcher, Query q,
+ String sQuery, Locale locale,
+ StringWriter result) throws IOException {
+ // Sorted search
+ long firstSearch = -System.currentTimeMillis();
+ ExposedFieldComparatorSource exposedFCS =
+ new ExposedFieldComparatorSource(searcher.getIndexReader(), locale);
+ Sort sort = new Sort(new SortField("id", exposedFCS));
+ TopFieldDocs docs = searcher.search(q, null, 5, sort);
+ firstSearch += System.currentTimeMillis();
+
+ result.append("First natural order sorted search for \""+ sQuery
+ + "\" with " + docs.totalHits + " hits: "
+ + getTime(firstSearch) + "\n");
+
+ long subSearchMS = 0;
+ final int RUNS = 5;
+ final int MAXHITS = 5;
+ TopFieldDocs topDocs = null;
+ for (int i = 0 ; i < RUNS ; i++) {
+ subSearchMS -= System.currentTimeMillis();
+ topDocs = searcher.search(q.weight(searcher), null, MAXHITS, sort, true);
+ subSearchMS += System.currentTimeMillis();
+ }
+
+ result.append("Subsequent " + RUNS
+ + " sorted searches average response time: "
+ + getTime(subSearchMS / RUNS) + "\n");
+ for (int i = 0 ; i < Math.min(topDocs.totalHits, MAXHITS) ; i++) {
+ int docID = topDocs.scoreDocs[i].doc;
+ result.append(String.format(
+ "Hit #%d was doc #%d with id %s\n",
+ i, docID,
+ ((BytesRef)((FieldDoc)topDocs.scoreDocs[i]).fields[0]).
+ utf8ToString()));
+ }
+ }
+
+ private String getTime(long ms) {
+ if (ms < 2999) {
+ return ms + " ms";
+ }
+ long seconds = Math.round(ms / 1000.0);
+ long minutes = seconds / 60;
+ return String.format("%d:%02d minutes", minutes, seconds - (minutes * 60));
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,126 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Single source and single field Iterator that provides terms and potentially
+ * document IDs.
+ *
+ * It is recommended to use a {@link CachedTermProvider} as source or otherwise
+ * ensure that ordered iteration is fast if no docIDs are to be returned..
+ * If docIDs are to be returned, caching at the source level will not help.
+ *
+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless
+ * {@link #setReuseTuple(boolean)} is called with false.
+ */
+public class TermDocIterator implements Iterator {
+ private final TermProvider source;
+ private final PackedInts.Reader order;
+ private final boolean collectDocIDs;
+
+ private DocsEnum docsEnum = null;
+
+ private int position = 0;
+ private boolean reuseTuple = true;
+
+ private boolean pending = false; // true == Terms or docIDs are ready in tuple
+ private final ExposedTuple tuple;
+
+ public TermDocIterator(TermProvider source, boolean collectDocIDs)
+ throws IOException {
+ this.source = source;
+ this.collectDocIDs = collectDocIDs;
+ order = source.getOrderedOrdinals();
+ String field = source.getField(0);
+
+ tuple = new ExposedTuple(field, null, 0, 0, -1);
+ }
+
+ // Ensure that we can deliver an id
+ public boolean hasNext() {
+ while (true) {
+ if (pending) {
+ return true;
+ }
+ if (position >= order.size()) {
+ if (source instanceof CachedTermProvider) {
+ System.out.println("TermDocsIterator depleted: "
+ + ((CachedTermProvider)source).getStats());
+ }
+ return false;
+ }
+ try {
+ seekToNextTerm();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next docID", e);
+ }
+ }
+ }
+
+ public ExposedTuple next() {
+ if (!hasNext()) {
+ throw new IllegalStateException("The iterator is depleted");
+ }
+ if (!collectDocIDs) {
+ pending = false;
+ } else {
+ tuple.docID = docsEnum.docID();
+ try {
+ if (docsEnum.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
+ pending = false;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next document", e);
+ }
+ }
+ return reuseTuple ? tuple : new ExposedTuple(tuple);
+ }
+
+ private void seekToNextTerm() throws IOException {
+ while (true) {
+ pending = false;
+ if (position >= order.size()) {
+ return;
+ }
+ tuple.indirect = position;
+ tuple.ordinal = order.get(position);
+ try {
+ tuple.term = source.getTerm(tuple.ordinal);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to resolve the term at order="
+ + order + ", ordinal=" + order.get(position), e);
+ }
+ position++;
+
+ if (!collectDocIDs) {
+ pending = true;
+ return;
+ }
+
+ docsEnum = source.getDocsEnum(tuple.ordinal, docsEnum);
+ int doc;
+ if ((doc = docsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS
+ && !(source.getReader().hasDeletions()
+ && source.getReader().getDeletedDocs().get(doc))) {
+ pending = true;
+ return; // We break at first genuine docID
+ }
+ }
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("Not a valid operation");
+ }
+
+ public void setReuseTuple(boolean reuseTuple) {
+ this.reuseTuple = reuseTuple;
+ }
+}
Index: lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java
===================================================================
--- lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,47 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestGrowingMutable extends LuceneTestCase {
+
+ public void testMiscGrowth() throws Exception {
+ GrowingMutable g = new GrowingMutable();
+
+ g.set(100, 1000);
+ assertEquals("The value at index 100 should be correct", 1000, g.get(100));
+ assertEquals("The size should be correct", 1, g.size());
+
+ g.set(98, 5000);
+ assertEquals("The value at index 98 should still be correct",
+ 5000, g.get(98));
+ assertEquals("The value at index 100 should still be correct",
+ 1000, g.get(100));
+ assertEquals("The size should be extended", 3, g.size());
+
+ g.set(-123, -12345);
+ assertEquals("The value at index 98 should still be correct",
+ 5000, g.get(98));
+ assertEquals("The value at index 100 should still be correct",
+ 1000, g.get(100));
+ assertEquals("The value at -123 should be as defined",
+ -12345, g.get(-123));
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetMap.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,189 @@
+package org.apache.lucene.search.exposed.facet;
+
+import org.apache.lucene.search.exposed.ExposedSettings;
+import org.apache.lucene.search.exposed.ExposedTuple;
+import org.apache.lucene.search.exposed.TermProvider;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Provides a map from docID to tagIDs (0 or more / document), where tagID
+ * refers to a tag one out of multiple TermProviders. The mapping is optimized
+ * towards low memory footprint and extraction of all tags for each given
+ * document ID.
+ */
+public class FacetMap {
+ private final List providers;
+ private final int[] indirectStarts;
+ // TODO: Make this final by returning a Pair from extractTags
+ private PackedInts.Mutable doc2ref;
+ private final PackedInts.Mutable refs;
+
+ public FacetMap(int docCount, List providers)
+ throws IOException {
+ this.providers = providers;
+ indirectStarts = new int[providers.size() +1];
+ int start = 0;
+ for (int i = 0 ; i < providers.size() ; i++) {
+ indirectStarts[i] = start;
+ start += providers.get(i).getUniqueTermCount();
+ }
+ indirectStarts[indirectStarts.length-1] = start;
+// doc2ref = PackedInts.getMutable(docCount+1, PackedInts.bitsRequired(start));
+ refs = extractTags(docCount);
+ }
+
+ public int getTagCount() {
+ return indirectStarts[indirectStarts.length-1];
+ }
+
+ /*
+ In order to efficiently populate the ref-structure, we perform a three-pass
+ run.
+ Pass 1: The number of references is counted for each document.
+ Pass 2: The doc2ref and refs-arrays are initialized so that doc2ref points
+ to the final offsets in refs and so that the first entry in each refs-chunk
+ designate the offset in the chunk for the next reference to store.
+ Pass 3: The refs-array is filled by iterating the ExposedTuples for each
+ TermProvider, adjusting the docID accordingly and storing the tagID in the
+ refs-array at the position given by doc2ref plus the offset from pass 2.
+ If the offset was larger than 0, it it decreased.
+ */
+ private PackedInts.Mutable extractTags(int docCount) throws IOException {
+ // We start by counting the references as this spares us a lot of array
+ // content re-allocation
+ int[] tagCounts = new int[docCount]; // One counter for each doc
+ long totalRefs = 0;
+ {
+ for (TermProvider provider: providers) {
+ final Iterator tuples = provider.getIterator(true);
+ while (tuples.hasNext()) {
+ final long docID = tuples.next().docID;
+ if (docID != -1) {
+ tagCounts[((int)docID)]++;
+ }
+ }
+ }
+ for (int count: tagCounts) {
+ totalRefs += count;
+ }
+ if (totalRefs > Integer.MAX_VALUE) {
+ throw new IllegalStateException(
+ "The current implementations does not support more that " +
+ "Integer.MAX_VALUE references to tags. The number of " +
+ "references was " + totalRefs);
+ }
+ }
+ doc2ref = ExposedSettings.getMutable(docCount+1, (int)totalRefs);
+
+ // Now we know the number of references so we create the refs-array
+ // and fill the combined doc2ref and refs with the doc to tag references
+ PackedInts.Mutable refs = ExposedSettings.getMutable(
+ (int)totalRefs, getTagCount());
+ {
+ int offset = 0;
+ for (int i = 0 ; i < tagCounts.length ; i++) {
+ doc2ref.set(i, offset);
+ if (tagCounts[i] != 0) {
+ // TODO: Check if this trick is safe. Can we encounter overflow?
+ // First bet is no as the number of references from a single doc
+ // cannot exceed the total number of tags
+ refs.set(offset, tagCounts[i]-1);
+ offset += tagCounts[i];
+ }
+ }
+ doc2ref.set(doc2ref.size()-1, offset);
+ }
+
+ // We are now ready to fill in the actual tagIDs. There will be a lot of
+ // random writes to the refs-array as we're essentially inverting index
+ // and value from the TermDocs.
+ for (int providerNum = 0 ; providerNum < providers.size() ; providerNum++) {
+ final TermProvider provider = providers.get(providerNum);
+ final long termOffset = indirectStarts[providerNum];
+ final Iterator tuples = provider.getIterator(true);
+ while (tuples.hasNext()) {
+ final ExposedTuple tuple = tuples.next();
+ final long docID = tuple.docID;
+ if (docID == -1) { // It happens sometimes with non-expunged deletions
+ continue;
+ }
+ final int refsOrigo = (int)doc2ref.get((int)docID);
+
+ final int chunkOffset = (int)refs.get(refsOrigo);
+ try {
+ refs.set(refsOrigo + chunkOffset, tuple.indirect + termOffset);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new RuntimeException(
+ "Array index out of bounds. refs.size=" + refs.size()
+ + ", refs.bitsPerValue=" + refs.getBitsPerValue()
+ + ", refsOrigo+chunkOffset="
+ + refsOrigo + "+" + chunkOffset
+ + "=" + (refsOrigo+chunkOffset)
+ + ", tuple.indirect+termOffset="
+ + tuple.indirect + "+" + termOffset + "="
+ + (tuple.indirect+termOffset), e);
+ }
+ if (chunkOffset != 0) {
+ refs.set(refsOrigo, chunkOffset-1);
+ }
+ }
+ }
+ return refs;
+ }
+
+ /**
+ * Takes an array where each entry corresponds to a tagID in this facet map
+ * and increments the counts for the tagIDs associated with the given docID.
+ * @param tagCounts a structure for counting occurences of tagIDs.
+ * @param docID an absolute document ID from which to extract tagIDs.
+ */
+ // TODO: Check if static helps speed in this inner loop method
+ public void updateCounter(final int[] tagCounts, final int docID) {
+ for (int refIndex = (int)doc2ref.get(docID);
+ refIndex < doc2ref.get(docID+1) ; refIndex++) {
+ tagCounts[(int)refs.get(refIndex)]++;
+ }
+ }
+
+ public int[] getIndirectStarts() {
+ return indirectStarts;
+ }
+
+ public List getProviders() {
+ return providers;
+ }
+
+ public BytesRef getOrderedTerm(final int termIndirect) throws IOException {
+ for (int i = 0 ; i < providers.size() ; i++) {
+ if (termIndirect < indirectStarts[i+1]) {
+ return providers.get(i).getOrderedTerm(
+ termIndirect- indirectStarts[i]);
+ }
+ }
+ throw new ArrayIndexOutOfBoundsException(
+ "The indirect " + termIndirect + " was too high. The maximum indirect "
+ + "supported by the current map is "
+ + indirectStarts[indirectStarts.length-1]);
+ }
+
+ public String toString() {
+ StringWriter sw = new StringWriter();
+ sw.append("FacetMap(#docs=").append(Integer.toString(doc2ref.size()-1));
+ sw.append(", refs=").append(Integer.toString(refs.size()));
+ sw.append(", providers(");
+ for (int i = 0 ; i < providers.size() ; i++) {
+ if (i != 0) {
+ sw.append(", ");
+ }
+ sw.append(providers.get(i).toString());
+ }
+ sw.append("))");
+ return sw.toString();
+ }
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/ExposedHelper.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/ExposedHelper.java Wed Sep 22 11:44:46 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/ExposedHelper.java Wed Sep 22 11:44:46 CEST 2010
@@ -0,0 +1,166 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.Collator;
+import java.text.DecimalFormat;
+import java.util.*;
+
+public class ExposedHelper {
+ static final char[] CHARS = // Used for random content
+ ("abcdefghijklmnopqrstuvwxyzæøåABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ" +
+ "1234567890 ").toCharArray();
+ public static final File INDEX_LOCATION =
+ new File("tmp/testfieldtermprovider");
+
+ // Fields in the test index
+ public static final String ID = "id"; // doc #0 = "0", doc #1 = "1" etc
+ public static final String EVEN = "even"; // "true" or "false"
+ public static final String ALL = "all"; // all == "all"
+ public static final String EVEN_NULL = "evennull"; // odd = random content
+ public static final String MULTI = "facet"; // 0-5 of values A to Z
+
+ public static final DecimalFormat ID_FORMAT = new DecimalFormat("00000000");
+
+ public ExposedHelper() {
+ deleteIndex();
+ INDEX_LOCATION.mkdirs();
+ }
+
+ public void close() {
+ deleteIndex();
+ }
+ private void deleteIndex() {
+ if (INDEX_LOCATION.exists()) {
+ for (File file: INDEX_LOCATION.listFiles()) {
+ file.delete();
+ }
+ INDEX_LOCATION.delete();
+ }
+ }
+
+ public static class Pair implements Comparable {
+ public long docID;
+ public String term;
+ public Collator comparator;
+
+ public Pair(long docID, String term, Collator comparator) {
+ this.docID = docID;
+ this.term = term;
+ this.comparator = comparator;
+ }
+
+ public int compareTo(Pair o) {
+ return comparator.compare(term, o.term);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof Pair)) {
+ return false;
+ }
+ Pair other = (Pair)obj;
+ return term.equals(other.term) && docID == other.docID;
+ }
+
+ @Override
+ public String toString() {
+ return "Pair(" + docID + ", " + term + ")";
+ }
+ }
+
+ public void createIndex(int docCount, List fields,
+ int fieldContentLength, int minSegments)
+ throws IOException {
+ long startTime = System.nanoTime();
+ File location = INDEX_LOCATION;
+ Random random = new Random(87);
+
+ Directory dir = FSDirectory.open(INDEX_LOCATION);
+ IndexWriter writer = new IndexWriter(
+ dir, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT,
+ new MockAnalyzer()));
+
+ writer.getConfig().setRAMBufferSizeMB(16.0);
+ for (int docID = 0 ; docID < docCount ; docID++) {
+ Document doc = new Document();
+ for (String field: fields) {
+ doc.add(new Field(
+ field,
+ field + "_" + getRandomString(
+ random, CHARS, 1, fieldContentLength) + docID,
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ }
+ doc.add(new Field(ID, ID_FORMAT.format(docID),
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.add(new Field(EVEN, docID % 2 == 0 ? "true" : "false",
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ int facets = random.nextInt(6);
+ for (int i = 0 ; i < facets ; i++) {
+ doc.add(new Field(MULTI,
+ Character.toString((char)(random.nextInt(25) + 'A')),
+ Field.Store.NO, Field.Index.NOT_ANALYZED));
+ }
+ if (docID % 2 == 1) {
+ doc.add(new Field(EVEN_NULL, getRandomString(
+ random, CHARS, 1, fieldContentLength) + docID,
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ }
+ doc.add(new Field(ALL, ALL, Field.Store.YES, Field.Index.NOT_ANALYZED));
+ writer.addDocument(doc);
+ if (docID == docCount / minSegments) {
+ writer.commit(); // Ensure minSegments
+ }
+ }
+// writer.optimize();
+ writer.close();
+ System.out.println(String.format(
+ "Created %d document index with %d fields with average " +
+ "term length %d and total size %s in %sms at %s",
+ docCount, fields.size() + 2, fieldContentLength / 2,
+ readableSize(calculateSize(location)),
+ (System.nanoTime() - startTime) / 1000000, location.getAbsolutePath()));
+ }
+
+ private static StringBuffer buffer = new StringBuffer(100);
+ static synchronized String getRandomString(
+ Random random, char[] chars, int minLength, int maxLength) {
+ int length = minLength == maxLength ? minLength :
+ random.nextInt(maxLength-minLength+1) + minLength;
+ buffer.setLength(0);
+ for (int i = 0 ; i < length ; i++) {
+ buffer.append(chars[random.nextInt(chars.length)]);
+ }
+ return buffer.toString();
+ }
+
+ private String readableSize(long size) {
+ return size > 2 * 1048576 ?
+ size / 1048576 + "MB" :
+ size > 2 * 1024 ?
+ size / 1024 + "KB" :
+ size + "bytes";
+ }
+
+ private long calculateSize(File file) {
+ long size = 0;
+ if (file.isDirectory()) {
+ for (File sub: file.listFiles()) {
+ size += calculateSize(sub);
+ }
+ } else {
+ size += file.length();
+ }
+ return size;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedSettings.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedSettings.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedSettings.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,54 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.packed.PackedInts;
+
+public class ExposedSettings {
+ public static enum PRIORITY {memory, speed;
+ public static PRIORITY fromString(String p) {
+ if (memory.toString().equals(p)) {
+ return memory;
+ } else if (speed.toString().equals(p)) {
+ return speed;
+ }
+ throw new IllegalArgumentException(
+ "The String '" + p + "' was not recognized as a priority");
+ }
+ }
+ public static final PRIORITY DEFAULT_PRIORITY = PRIORITY.memory;
+
+ /**
+ * The priority determines what is most important: memory or speed.
+ * The setting is used whenever a PackedInts-structure is created to determine
+ * if rounding to byte, short, int or long representations should be used or
+ * if the optimal space saving representation should be used (no rounding).
+ *
+ * The speed gain depends very much on the actual index and the setup for
+ * sorting, faceting and index lookup, but a rough rule of thumb is that
+ * memory usage increases 25-50% and speed increases 25-50% for faceting and
+ * initial index look ups when speed is chosen.
+ */
+ public static PRIORITY priority = DEFAULT_PRIORITY;
+
+ /**
+ * Construct a PackedInts.Mutable based on the given values and the overall
+ * priority between memory usage and speed. If the maxValue is below 2^4 of
+ * between 2^32 and 2^40, a standard PackedInts.Mutable is created, else the
+ * maxValue is rounded up to 2^8-1, 2^16-1, 2^32-1 or 2^64-1;
+ * @param valueCount the number of values that the mutator should hold.
+ * @param maxValue the maximum value that will ever be inserted.
+ * @return a mutable optimized for either speed or memory.
+ */
+ public static PackedInts.Mutable getMutable(int valueCount, long maxValue) {
+ int bitsRequired = PackedInts.bitsRequired(maxValue);
+ switch (priority) {
+ case memory: return PackedInts.getMutable(valueCount, bitsRequired);
+ case speed: if (bitsRequired <= 4 ||
+ (bitsRequired > 32 && bitsRequired < 40)) { // TODO: Consider the 40
+ return PackedInts.getMutable(valueCount, bitsRequired);
+ } return PackedInts.getMutable(
+ valueCount, PackedInts.getNextFixedSize(bitsRequired));
+ default: throw new IllegalArgumentException(
+ "The priority " + priority + " is unknown");
+ }
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,15 @@
+package org.apache.lucene.search.exposed;
+
+public class ExposedUtil {
+ private ExposedUtil() {
+ // Disables all constructors
+ }
+
+ public static String time(String type, long num, long ms) {
+ if (ms == 0) {
+ return num + " " + type + " in 0 ms";
+ }
+ return num + " " + type + " in " + ms + " ms: ~= "
+ + num/ms + " " + type + "/ms";
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java Tue Sep 21 10:34:57 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java Tue Sep 21 10:34:57 CEST 2010
@@ -0,0 +1,254 @@
+/* $Id:$
+ *
+ * The Summa project.
+ * Copyright (C) 2005-2010 The State and University Library
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A request defines how a TermProvider is to behave with regard to fields
+ * and sorting.
+ *
+ * A FacetGroup is the primary access point for sorting. A FacetGroup contains one or
+ * more Fields from one or more segments.
+ *
+ * Field is single field and single segment oriented. It is used internally by
+ * FacetGroup but can be used externally if the index is fully optimized and sorting
+ * is requested for a single field.
+ */
+public class ExposedRequest {
+ /**
+ * If the comparator key is set to the value "LUCENE", the order of the
+ * terms is Lucene's default order.
+ */
+ public static final String LUCENE_ORDER = "LUCENE";
+ public static final String FREE_ORDER = "FREE";
+
+ /**
+ * This signifies that the order of the terms does not matter to the caller.
+ * The normal use-cache is for count-based faceting. By specifying WHATEVER
+ * as the order, the TermProvider cache is free to return any element that
+ * is based on the right fields.
+ */
+ public static final String WHATEVER_ORDER = "WHATEVER";
+
+ public static Group createGroup(
+ String name, List fieldNames, Comparator comparator,
+ String comparatorID) {
+ if (fieldNames.size() == 0) {
+ throw new IllegalArgumentException("There must be at least 1 field name");
+ }
+ List fieldRequests =
+ new ArrayList(fieldNames.size());
+ for (String fieldName: fieldNames) {
+ fieldRequests.add(new ExposedRequest.Field(
+ fieldName, comparator, comparatorID));
+ }
+ return new ExposedRequest.Group(
+ name, fieldRequests, comparator, comparatorID);
+ }
+
+ /**
+ * A group is a collections of fields that are to be treated as one.
+ * An example could be a group named "title" with the fields "lti",
+ * "sort_title" and "subtitle" as fields. The same comparator must be used
+ * for all fields in order for term ordering to be consistent for the group.
+ */
+ public static class Group {
+ private String name;
+ private List fields;
+ private Comparator comparator;
+ private String comparatorID;
+
+ public Group(String name, List fields,
+ Comparator comparator, String comparatorID) {
+ this.name = name;
+ this.fields = fields;
+ this.comparator = comparator;
+ this.comparatorID = comparatorID;
+ }
+
+ /**
+ * Checks whether the given FacetGroup is equal to . The order of the contained
+ * fields is irrelevant, but the set of fields must be equal.
+ *
+ * The name of the group is not used while comparing.
+ * @param other the group to compare to.
+ * @return true if the two groups are equivalent..
+ */
+ // TODO: Fix hashcode to fulfill the equals vs. hashcode contract
+ public boolean equals(Group other) {
+ if (other.getFields().size() != getFields().size() ||
+ !getComparatorID().equals(other.getComparatorID())) {
+ return false;
+ }
+
+ fieldLoop:
+ for (Field otherField: other.getFields()) {
+ for (Field thisField: getFields()) {
+ if (thisField.equals(otherField)) {
+ continue fieldLoop;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Similar to {@link #equals(Group)} but if the comparatorID for the other
+ * group is {@link #FREE_ORDER}, it matches any comparatorID this group has.
+ * @param other the group to check with.
+ * @return true if this group can deliver data for the other group.
+ */
+ public boolean worksfor(Group other) {
+ if (other.getFields().size() != getFields().size()) {
+ return false;
+ }
+ if (!FREE_ORDER.equals(other.getComparatorID()) &&
+ !getComparatorID().equals(other.getComparatorID())) {
+ return false;
+ }
+
+ fieldLoop:
+ for (Field otherField: other.getFields()) {
+ for (Field thisField: getFields()) {
+ if (thisField.worksfor(otherField)) {
+ continue fieldLoop;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
+ public String getName() {
+ return name;
+ }
+ public List getFields() {
+ return fields;
+ }
+ public Comparator getComparator() {
+ return comparator;
+ }
+ public String getComparatorID() {
+ return comparatorID;
+ }
+
+ public List getFieldNames() {
+ List fieldNames = new ArrayList(fields.size());
+ for (Field field: fields) {
+ fieldNames.add(field.getField());
+ }
+ return fieldNames;
+ }
+
+ /**
+ * If the comparatorID is {@link #FREE_ORDER} we set it to
+ * {@link #LUCENE_ORDER};
+ */
+ void normalizeComparatorIDs() {
+ if (!FREE_ORDER.equals(getComparatorID())) {
+ return;
+ }
+ comparatorID = FREE_ORDER;
+ for (Field field: fields) {
+ field.comparatorID = FREE_ORDER;
+ }
+ }
+ }
+
+ public static class Field {
+ private String field;
+ private Comparator comparator;
+ private String comparatorID;
+
+ public Field(
+ String field, Comparator comparator, String comparatorID) {
+ this.field = field;
+ this.comparator = comparator;
+ this.comparatorID = comparatorID;
+ }
+
+ /**
+ * Checks whether the given object is also a Field and if so whether the
+ * field and comparatorIDs are equal.
+ * @param obj candidate for equality.
+ * @return true if the given object is equivalent to this.
+ */
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Field)) {
+ return false;
+ }
+ Field other = (Field)obj;
+ return field.equals(other.field)
+ && comparatorID.equals(other.comparatorID);
+ }
+
+ /**
+ * Similar to {@link #equals} but if the comparatorID for the other
+ * field is {@link #FREE_ORDER}, it matches any comparatorID this field has.
+ * @param other the field to check with.
+ * @return true if this group can deliver data for the other field.
+ **/
+ public boolean worksfor(Field other) {
+ return field.equals(other.field)
+ && (FREE_ORDER.equals(other.comparatorID)
+ ||comparatorID.equals(other.comparatorID));
+ }
+
+ public String getField() {
+ return field;
+ }
+ public Comparator getComparator() {
+ return comparator;
+ }
+ public String getComparatorID() {
+ return comparatorID;
+ }
+
+ public boolean equals(Field otherField) {
+ return getField().equals(otherField.getField()) &&
+ getComparatorID().equals(otherField.getComparatorID());
+ }
+
+ }
+
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java Sat Sep 18 00:02:00 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java Sat Sep 18 00:02:00 CEST 2010
@@ -0,0 +1,197 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+public class TestGroupTermProvider extends LuceneTestCase {
+ public static final int DOCCOUNT = 10;
+ private ExposedHelper helper;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ // TODO: Figure out how to force Flex the clean way
+ CodecProvider.setDefaultCodec("Standard");
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ helper.close();
+ }
+
+ public void testIndexGeneration() throws Exception {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ long termCount = 0;
+ TermsEnum terms = MultiFields.getFields(reader).
+ terms(ExposedHelper.ID).iterator();
+ while (terms.next() != null) {
+ assertEquals("The ID-term #" + termCount + " should be correct",
+ ExposedHelper.ID_FORMAT.format(termCount),
+ terms.term().utf8ToString());
+ termCount++;
+ }
+
+ assertEquals("There should be the right number of terms",
+ DOCCOUNT, termCount);
+ reader.close();
+ }
+
+ public void testOrdinalAccess() throws IOException {
+ String[] fieldNames = new String[]{"a"};
+ helper.createIndex( DOCCOUNT, Arrays.asList(fieldNames), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ for (String fieldName: fieldNames) {
+ // FIXME: The ordinals are not comparable to the exposed ones!
+ TermsEnum terms =
+ MultiFields.getFields(reader).terms(fieldName).iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ }
+ }
+
+ TermProvider groupProvider = ExposedFactory.createProvider(
+ reader, "TestGroup", Arrays.asList(fieldNames), null, "null");
+
+ ArrayList exposedOrdinals = new ArrayList(DOCCOUNT);
+ for (int i = 0 ; i < groupProvider.getOrdinalTermCount() ; i++) {
+ exposedOrdinals.add(groupProvider.getTerm(i).utf8ToString());
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedOrdinals.size());
+ // We sort as the order of the terms is not significant here
+ Collections.sort(plainExtraction);
+ Collections.sort(exposedOrdinals);
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedOrdinals.get(i));
+ }
+ reader.close();
+
+ }
+ public void testTermSortAllDefined() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ testTermSort(reader, Arrays.asList("a"));
+ reader.close();
+ }
+
+ public void testTermSortAllScarce() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ testTermSort(reader, Arrays.asList("even"));
+ reader.close();
+ }
+
+ private void testTermSort(IndexReader index, List fieldNames)
+ throws IOException {
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ // TODO: Make this handle multiple field names
+ TermsEnum terms =
+ MultiFields.getFields(index).terms(fieldNames.get(0)).iterator();
+ while (terms.next() != null) {
+ String next = terms.term().utf8ToString();
+ plainExtraction.add(next);
+// System.out.println("Default order term #"
+// + (plainExtraction.size() - 1) + ": " + next);
+ }
+ Collections.sort(plainExtraction, sorter);
+
+ TermProvider groupProvider = ExposedFactory.createProvider(
+ index, "a-group", fieldNames,
+ ExposedComparators.collatorToBytesRef(sorter), "da");
+
+ ArrayList exposedExtraction = new ArrayList(DOCCOUNT);
+ Iterator ei = groupProvider.getIterator(false);
+ int count = 0;
+ while (ei.hasNext()) {
+ String next = ei.next().term.utf8ToString();
+ exposedExtraction.add(next);
+// System.out.println("Exposed sorted term #" + count++ + ": " + next);
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedExtraction.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedExtraction.get(i));
+ }
+ }
+
+ public void testDocIDMapping() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plain =
+ new ArrayList(DOCCOUNT);
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ plain.add(new ExposedHelper.Pair(
+ docID, reader.document(docID).get("a"), sorter));
+// System.out.println("Plain access added " + plain.get(plain.size()-1));
+ }
+ Collections.sort(plain);
+
+ TermProvider index =
+ ExposedFactory.createProvider(reader, "docIDGroup", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(sorter), "foo");
+
+ ArrayList exposed =
+ new ArrayList(DOCCOUNT);
+ Iterator ei = index.getIterator(true);
+
+ while (ei.hasNext()) {
+ ExposedTuple next = ei.next();
+// System.out.println("Extracted exposed " + next);
+ exposed.add(new ExposedHelper.Pair(
+ next.docID, next.term.utf8ToString(), sorter));
+ }
+ Collections.sort(exposed);
+
+ assertEquals("The two docID->term maps should be of equal length",
+ plain.size(), exposed.size());
+ for (int i = 0 ; i < plain.size() ; i++) {
+// System.out.println("Sorted docID, term #" + i + ". Plain=" + plain.get(i)
+// + ", Exposed=" + exposed.get(i));
+ assertEquals("Mapping #" + i + " should be equal",
+ plain.get(i), exposed.get(i));
+ }
+ reader.close();
+ }
+
+ public void testTermCount() throws IOException {
+ helper.createIndex(100, Arrays.asList("a"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ ExposedHelper.MULTI, null, ExposedRequest.LUCENE_ORDER);
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group("TestGroup",
+ Arrays.asList(fieldRequest), null, ExposedRequest.LUCENE_ORDER);
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, groupRequest);
+
+ assertEquals("The number of unique terms for multi should match",
+ 25, provider.getUniqueTermCount());
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,253 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Keeps track of a group containing 1 or more providers ensuring term
+ * uniqueness and ordering. The providers are normally either
+ * {@link FieldTermProvider} or {@link GroupTermProvider}.
+ *
+ * The standard use case calls for caching the GroupTermProvider at segment
+ * level as group setups rarely changes.
+ */
+// TODO: Extend to handle term ordinals > Integer.MAX_VALUE
+public class GroupTermProvider extends TermProviderImpl {
+ private List providers;
+ private ExposedRequest.Group request;
+ private final int readerHash;
+
+ private PackedInts.Reader order;
+
+ private long[] termOrdinalStarts; // Starting points for term ordinals in the
+ // providers list
+ // FIXME: this should be relative to segments, not providers.
+ //Current implementation is not valid for multiple fields in the same segment
+ private long[] docIDStarts;
+
+ public GroupTermProvider(int readerHash,
+ List providers, ExposedRequest.Group request,
+ boolean cacheTables) throws IOException {
+ super(request.getComparator(), request.getComparatorID(),
+ "Group " + request.getName(), cacheTables);
+ this.readerHash = readerHash;
+ this.providers = providers;
+ this.request = request;
+
+ calculateStarts();
+ }
+
+ /**
+ * The starts are straight forward for term ordinals as they are just mapped
+ * as one long list of ordinals in the source order. We can do there is no
+ * authoritative index-level ordinal mapping.
+ *
+ * Document IDs are a bit trickier as they are used at the index-level.
+ * We need to handle the case where more than one field from the same segment
+ * make up a group. In that case the docIDs should have the same start.
+ *
+ * Important: In order for the calculator to work, the sources must be
+ * monotonic increasing
+ * @throws IOException if the underlying providers failed.
+ */
+ private void calculateStarts() throws IOException {
+ long sanityCheck = 0;
+ termOrdinalStarts = new long[providers.size() + 1];
+ docIDStarts = new long[providers.size() + 1];
+
+ for (int i = 1 ; i <= providers.size() ; i++) {
+ termOrdinalStarts[i] =
+ termOrdinalStarts[i-1] + providers.get(i-1).getOrdinalTermCount();
+ sanityCheck += providers.get(i-1).getOrdinalTermCount();
+ }
+ if (sanityCheck > Integer.MAX_VALUE-1) {
+ throw new InternalError("There are " + sanityCheck + " terms in total in "
+ + "the underlying TermProviders which is more than the current limit "
+ + "of Integer.MAX_VALUE");
+ }
+
+ long lastHash = -1;
+ long docStart = 0;
+ for (int i = 0 ; i < providers.size() ; i++) {
+ docIDStarts[i] = docStart;
+ TermProvider provider = providers.get(i);
+ docStart += provider.getRecursiveHash() == lastHash ? 0 :
+ provider.getMaxDoc();
+ lastHash = provider.getRecursiveHash();
+ }
+ docIDStarts[docIDStarts.length-1] = docStart;
+ }
+
+ public ExposedRequest.Group getRequest() {
+ return request;
+ }
+
+ public String getField(long ordinal) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getField(adjustOrdinal(ordinal, providerIndex));
+ }
+
+ private int getProviderIndex(long ordinal) throws IOException {
+ for (int i = 1 ; i < termOrdinalStarts.length ; i++) {
+ if (ordinal < termOrdinalStarts[i]) {
+ return i-1;
+ }
+ }
+ throw new IllegalArgumentException("The term ordinal " + ordinal
+ + " is above the maximum " + (getOrdinalTermCount() - 1));
+ }
+
+ private long adjustOrdinal(long ordinal, int providerIndex) {
+ return providerIndex == 0 ? ordinal :
+ ordinal - termOrdinalStarts[providerIndex];
+ }
+
+ public synchronized BytesRef getTerm(long ordinal) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getTerm(adjustOrdinal(ordinal, providerIndex));
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getDocsEnum(adjustOrdinal(ordinal, providerIndex), reuse);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return getField(getOrderedOrdinals().get((int)indirect));
+ }
+
+ public BytesRef getOrderedTerm(final long indirect) throws IOException {
+ return indirect == -1 ? null :
+ getTerm(getOrderedOrdinals().get((int)indirect));
+ }
+
+ /**
+ * Note that this method calculates ordered ordinals to determine the unique
+ * terms.
+ * @return The number of unique terms.
+ * @throws IOException if the underlyind providers failed.
+ */
+ public long getUniqueTermCount() throws IOException {
+ return getOrderedOrdinals().size();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return termOrdinalStarts[termOrdinalStarts.length-1];
+ }
+
+ public long getMaxDoc() {
+ return docIDStarts[docIDStarts.length-1];
+ }
+
+ public IndexReader getReader() throws IOException {
+ throw new UnsupportedOperationException(
+ "Cannot request a reader from a collection of readers");
+ }
+
+ public int getReaderHash() {
+ return readerHash;
+ }
+
+ public int getRecursiveHash() {
+ int hash = 0;
+ for (TermProvider provider: providers) {
+ hash += provider.hashCode();
+ }
+ return hash;
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ if (order != null) {
+ return order;
+ }
+ PackedInts.Reader newOrder;
+/* if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID())) {
+ // TODO: This produces duplicates! We need to sort
+ newOrder = new IdentityReader((int)getOrdinalTermCount());
+ } else {*/
+ newOrder = sortOrdinals();
+ //}
+ if (cacheTables) {
+ order = newOrder;
+ }
+ return newOrder;
+ }
+
+ private PackedInts.Reader sortOrdinals() throws IOException {
+// System.out.println("FacetGroup sorting ordinals from " + providers.size()
+// + " providers");
+ int maxTermCount = (int)termOrdinalStarts[termOrdinalStarts.length-1];
+ long iteratorConstruction = System.currentTimeMillis();
+ PackedInts.Mutable collector = ExposedSettings.getMutable(
+ maxTermCount, maxTermCount);
+ //new GrowingMutable(0, maxTermCount, 0, maxTermCount);
+ Iterator iterator = getIterator(false);
+ iteratorConstruction = System.currentTimeMillis() - iteratorConstruction;
+ System.out.println("Group total iterator construction: "
+ + ExposedUtil.time("ordinals", maxTermCount, iteratorConstruction));
+
+ int uniqueTermCount = 0;
+ long extractionTime = System.currentTimeMillis();
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+// System.out.println("sortOrdinals " + tuple + " term = " + tuple.term.utf8ToString() + " lookup term " + getTerm(tuple.ordinal).utf8ToString());
+ collector.set((int)tuple.indirect, tuple.ordinal);
+ uniqueTermCount++;
+// collector.set(uniqueTermCount++, tuple.indirect);
+ }
+// System.out.println("Sorted merged term ordinals to " + collector);
+ PackedInts.Mutable result = ExposedSettings.getMutable(
+ uniqueTermCount, maxTermCount);
+ for (int i = 0 ; i < uniqueTermCount ; i++) {
+ result.set(i, collector.get(i));
+ }
+ extractionTime = System.currentTimeMillis() - extractionTime;
+ System.out.println("Group ordinal iterator depletion from "
+ + providers.size() + " providers: "
+ + ExposedUtil.time("ordinals", result.size(), extractionTime));
+ return result;
+ }
+
+
+ public Iterator getIterator(boolean collectDocIDs)
+ throws IOException {
+ return new MergingTermDocIterator(
+ this, providers, request.getComparator(), collectDocIDs);
+ }
+
+ public long segmentToIndexDocID(int providerIndex, long segmentDocID) {
+ return docIDStarts[providerIndex] + segmentDocID;
+ }
+
+ public long segmentToIndexTermOrdinal(
+ int providerIndex, long segmentTermOrdinal) {
+ return termOrdinalStarts[providerIndex] + segmentTermOrdinal;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.java Wed Sep 22 14:37:57 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.java Wed Sep 22 14:37:57 CEST 2010
@@ -0,0 +1,525 @@
+package org.apache.lucene.search.exposed.facet;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.exposed.ExposedCache;
+import org.apache.lucene.search.exposed.ExposedComparators;
+import org.apache.lucene.search.exposed.ExposedRequest;
+import org.apache.lucene.util.BytesRef;
+
+import javax.xml.stream.*;
+import java.io.*;
+import java.text.Collator;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * A specifications of groups of fields with sort order, max tags etc. Used for
+ * specifying how to generate the internal facet structure as well as the
+ * concrete user-request. The vocabulary is taken primarily from
+ * http://wiki.apache.org/solr/SimpleFacetParameters
+ *
+ * Sample: stringQuery(cheese), groups(
+ * title(fields(mytitle, subtitle), sort(index), comparator(lang=da),
+ * mincount(0), offset(-5), prefix(foo)),
+ * event(fields(event), mincount(1))
+ * )
+ * Escape parenthesis with }\) or \(, escape \ with \\.
+ */
+public class FacetRequest {
+ public static final String NAMESPACE =
+ "http://lucene.apache.org/exposed/facet/request/1.0";
+
+ public static final String ORDER_COUNT = "count";
+ public static final String ORDER_INDEX = "index";
+ public static final String ORDER_LOCALE = "locale";
+
+ public static final String DEFAULT_ORDER = ORDER_COUNT;
+ public static final int DEFAULT_MAXTAGS = 20;
+ public static final int DEFAULT_MINCOUNT = 0;
+ public static final int DEFAULT_OFFSET = 0;
+ public static final String DEFAULT_PREFIX = "";
+
+ private static final XMLInputFactory xmlFactory;
+ static {
+ xmlFactory = XMLInputFactory.newFactory();
+ xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, true);
+ }
+ private static final XMLOutputFactory xmlOutFactory;
+ static {
+ xmlOutFactory = XMLOutputFactory.newFactory();
+ }
+
+ // groups: group*
+ // group: fieldName* sort offset limit mincount prefix
+ // sort: count, index, custom (comparator)
+ private String stringQuery;
+ private final List groups;
+
+ private String order = DEFAULT_ORDER;
+ private String locale = null;
+ private int maxTags = DEFAULT_MAXTAGS;
+ private int minCount = DEFAULT_MINCOUNT;
+ private int offset = DEFAULT_OFFSET;
+ private String prefix = DEFAULT_PREFIX;
+
+
+ public FacetRequest(String stringQuery, List groups) {
+ this.stringQuery = stringQuery;
+ this.groups = groups;
+ }
+
+ /**
+ * @param request A facet request in XML as specified in FacetRequest.xsd.
+ * @return a facet request in Object-form.
+ * @throws XMLStreamException if the request could not be parsed.
+ */
+ public static FacetRequest parseXML(String request)
+ throws XMLStreamException {
+ XMLStreamReader reader =
+ xmlFactory.createXMLStreamReader(new StringReader(request));
+ List groups = null;
+
+ String stringQuery = null;
+ String order = DEFAULT_ORDER;
+ String locale = null;
+ int maxTags = DEFAULT_MAXTAGS;
+ int minCount = DEFAULT_MINCOUNT;
+ int offset = DEFAULT_OFFSET;
+ String prefix = DEFAULT_PREFIX;
+
+ reader.nextTag();
+ if (!atStart(reader, "facetrequest")) {
+ throw new XMLStreamException(
+ "Could not locate start tag in " + request);
+ }
+ for (int i = 0 ; i < reader.getAttributeCount() ; i++) {
+ String attribute = reader.getAttributeLocalName(i);
+ String value = reader.getAttributeValue(i);
+ if ("order".equals(attribute)) {
+ order = checkOrder(request, value);
+ } else if ("locale".equals(attribute)) {
+ locale = value;
+ } else if ("maxtags".equals(attribute)) {
+ maxTags = getInteger(request, "maxtags", value);
+ } else if ("mincount".equals(attribute)) {
+ minCount = getInteger(request, "mincount", value);
+ } else if ("offset".equals(attribute)) {
+ offset = getInteger(request, "offset", value);
+ } else if ("prefix".equals(attribute)) {
+ prefix = value;
+ }
+ }
+
+ while ((stringQuery == null || groups == null)
+ && reader.nextTag() != XMLStreamReader.END_DOCUMENT) {
+ if (atStart(reader, "query")) {
+ reader.next();
+ stringQuery = reader.getText();
+ reader.nextTag(); // END_ELEMENT
+ } else if (atStart(reader, "groups")) {
+ groups = resolveGroups(reader, request, order, locale, maxTags,
+ minCount, offset, prefix);
+ } else {
+ reader.nextTag(); // Ignore and skip to END_ELEMENT
+ }
+ }
+ FacetRequest fr = new FacetRequest(stringQuery, groups);
+ fr.setOrder(order);
+ fr.setLocale(locale);
+ fr.setMaxTags(maxTags);
+ fr.setMinCount(minCount);
+ fr.setOffset(offset);
+ fr.setPrefix(prefix);
+ return fr;
+ }
+
+ private static List resolveGroups(
+ XMLStreamReader reader, String request, String order,
+ String locale, int maxTags, int minCount, int offset, String prefix)
+ throws XMLStreamException {
+ List groups = new ArrayList();
+ while (reader.nextTag() != XMLStreamReader.END_DOCUMENT
+ && !atEnd(reader, "groups")) { // Not END_ELEMENT for groups
+ if (atStart(reader, "group")) {
+ groups.add(resolveGroup(
+ reader, request, order, locale, maxTags, minCount, offset, prefix));
+ } else {
+ reader.nextTag(); // Ignore and skip to END_ELEMENT
+ }
+ }
+ if (groups.size() == 0) {
+ throw new XMLStreamException("No groups defined for " + request);
+ }
+ return groups;
+ }
+
+ private static FacetGroup resolveGroup(
+ XMLStreamReader reader, String request, String order,
+ String locale, int maxTags, int minCount, int offset, String prefix)
+ throws XMLStreamException {
+ String name = null;
+ for (int i = 0 ; i < reader.getAttributeCount() ; i++) {
+ String attribute = reader.getAttributeLocalName(i);
+ String value = reader.getAttributeValue(i);
+ if ("order".equals(attribute)) {
+ order = checkOrder(request, value);
+ } else if ("locale".equals(attribute)) {
+ locale = value;
+ } else if ("maxtags".equals(attribute)) {
+ maxTags = getInteger(request, "maxtags", value);
+ } else if ("mincount".equals(attribute)) {
+ minCount = getInteger(request, "mincount", value);
+ } else if ("offset".equals(attribute)) {
+ offset = getInteger(request, "offset", value);
+ } else if ("prefix".equals(attribute)) {
+ prefix = value;
+ } else if ("name".equals(attribute)) {
+ name = value;
+ }
+ }
+ if (name == null) {
+ throw new XMLStreamException(
+ "FacetGroup name must be specified with the attribute 'name' in "
+ + request);
+ }
+ List fieldNames = new ArrayList();
+ reader.nextTag();
+ while (!atEnd(reader, "group")) {
+ if (atStart(reader, "fields")) {
+ reader.nextTag();
+ while (!atEnd(reader, "fields")) {
+ if (atStart(reader, "field")) {
+ String fieldName = null;
+ for (int i = 0 ; i < reader.getAttributeCount() ; i++) {
+ if ("name".equals(reader.getAttributeLocalName(i))) {
+ fieldName = reader.getAttributeValue(i);
+ }
+ }
+ if (fieldName == null) {
+ throw new XMLStreamException(
+ "Unable to determine name for field in group " + name
+ + " in " + request);
+ }
+ fieldNames.add(fieldName);
+ }
+ reader.nextTag(); // Until /fields
+ }
+ }
+ reader.nextTag(); // until /group
+ }
+ String comparatorID;
+ if (ORDER_COUNT.equals(order)) {
+ comparatorID = ExposedRequest.FREE_ORDER;
+ } else if (ORDER_INDEX.equals(order)) {
+ comparatorID = ExposedRequest.LUCENE_ORDER;
+ } else if (ORDER_LOCALE.equals(order)) {
+ comparatorID = new Locale(locale).toString();
+ } else {
+ throw new IllegalArgumentException(
+ "The order '" + order + "' is unknown");
+ }
+ ExposedRequest.Group groupRequest = ExposedRequest.createGroup(
+ name, fieldNames, createComparator(locale), comparatorID);
+ return new FacetGroup(
+ groupRequest, order, locale, offset, maxTags, minCount, prefix);
+ }
+
+ private static Comparator createComparator(String locale) {
+ if (locale == null || "".equals(locale)) {
+ return ExposedComparators.collatorToBytesRef(null);
+ }
+ return ExposedComparators.collatorToBytesRef(
+ Collator.getInstance(new Locale(locale)));
+ }
+
+ private static boolean atStart(XMLStreamReader reader, String tag) {
+ return reader.getEventType() == XMLStreamReader.START_ELEMENT
+ && tag.equals(reader.getLocalName())
+ && NAMESPACE.equals(reader.getNamespaceURI());
+ }
+
+ private static boolean atEnd(XMLStreamReader reader, String tag) {
+ return reader.getEventType() == XMLStreamReader.END_ELEMENT
+ && tag.equals(reader.getLocalName())
+ && NAMESPACE.equals(reader.getNamespaceURI());
+ }
+
+ private static int getInteger(String request, String attribute, String value)
+ throws XMLStreamException {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new XMLStreamException("The integer for " + attribute + " was '"
+ + value + "' which is not valid. Full request: " + request);
+ }
+ }
+
+ public static String checkOrder(String request, String order)
+ throws XMLStreamException {
+ if (ORDER_COUNT.equals(order) || ORDER_INDEX.equals(order)
+ || ORDER_LOCALE.equals(order)) {
+ return order;
+ }
+ throw new XMLStreamException("The order was '" + order + "' where only "
+ + ORDER_COUNT + ", " + ORDER_INDEX + " and " + ORDER_LOCALE
+ + " is allowed. Full request: " + request);
+ }
+
+ // TODO: Consider sorting the groups and fields to ensure compatibility
+ // with request where the order is changed
+ /**
+ * Produces a key created from the build-specific properties of the request.
+ * This includes stringQuery, group name, fields and comparator. It does not
+ * include extraction-specific things such as sort, mincount, offset and
+ * prefix. The key can be used for caching of the counts from a request.
+ * @return a key generated from the build-properties.
+ */
+ public String getBuildKey() {
+ StringWriter sw = new StringWriter();
+ sw.append("facetrequest(").append("query(").append(stringQuery);
+ sw.append("), ");
+ writeGroupKey(sw);
+ sw.append(")");
+ return sw.toString();
+ }
+
+ public String toXML() {
+ StringWriter sw = new StringWriter();
+ try {
+ XMLStreamWriter out = xmlOutFactory.createXMLStreamWriter(sw);
+ out.writeStartDocument("utf-8", "1.0");
+ out.writeCharacters("\n");
+ toXML(out);
+ out.writeEndDocument();
+ out.flush();
+ } catch (XMLStreamException e) { // Getting here means error in the code
+ throw new RuntimeException("Unable to create XML", e);
+ }
+ return sw.toString();
+ }
+ public void toXML(XMLStreamWriter out) throws XMLStreamException {
+ out.setDefaultNamespace(NAMESPACE);
+ out.writeStartElement("facetrequest");
+ out.writeDefaultNamespace(NAMESPACE);
+ out.writeAttribute("order", order);
+ if (locale != null) {
+ out.writeAttribute("locale", locale);
+ }
+ out.writeAttribute("maxtags", Integer.toString(maxTags));
+ out.writeAttribute("mincount", Integer.toString(minCount));
+ out.writeAttribute("offset", Integer.toString(offset));
+ out.writeAttribute("prefix", prefix);
+ out.writeCharacters("\n ");
+
+ out.writeStartElement("query");
+ out.writeCharacters(stringQuery);
+ out.writeEndElement();
+ out.writeCharacters("\n ");
+
+ out.writeStartElement("groups");
+ out.writeCharacters("\n");
+ for (FacetGroup group: groups) {
+ group.toXML(out);
+ }
+ out.writeCharacters(" ");
+ out.writeEndElement(); // groups
+ out.writeCharacters("\n");
+
+ out.writeEndElement(); // facetrequest
+ out.writeCharacters("\n");
+ out.flush();
+ }
+
+ /**
+ * Produces a key generated from the group-specific properties of the request.
+ * This includes group name, fields and comparator.
+ * The key is normally used for caching TermProviders.
+ * @return a key generated from the group-specific properties of the request.
+ */
+ public String getGroupKey() {
+ StringWriter sw = new StringWriter();
+ writeGroupKey(sw);
+ return sw.toString();
+ }
+
+ public List getGroups() {
+ return groups;
+ }
+
+ private void writeGroupKey(StringWriter sw) {
+ sw.append("groups(");
+ boolean first = true;
+ for (FacetGroup group: groups) {
+ if (first) {
+ first = false;
+ } else {
+ sw.append(", ");
+ }
+ sw.append(group.getBuildKey());
+ }
+ sw.append(")");
+ }
+
+ public static final class FacetGroup {
+ private final ExposedRequest.Group group;
+ private final String order;
+ private final String locale;
+ private final int offset;
+ private final int maxTags;
+ private final int minCount;
+ private final String prefix;
+ private final String buildKey;
+
+ public FacetGroup(ExposedRequest.Group group, String order, String locale,
+ int offset, int maxTags, int minCount, String prefix) {
+ this.group = group;
+ this.order = order;
+ this.locale = locale;
+ this.offset = offset;
+ this.maxTags = maxTags;
+ this.minCount = minCount;
+ this.prefix = prefix;
+ StringWriter sw = new StringWriter();
+ sw.append("group(name=").append(group.getName()).append(", order=");
+ sw.append(order).append(", locale=").append(locale).append(", fields(");
+ boolean first = true;
+ for (String field: group.getFieldNames()) {
+ if (first) {
+ first = false;
+ } else {
+ sw.append(", ");
+ }
+ sw.append(field);
+ }
+ sw.append("))");
+ buildKey = sw.toString();
+ }
+
+ public String getBuildKey() {
+ return buildKey;
+ }
+
+ public ExposedRequest.Group getGroup() {
+ return group;
+ }
+
+ public void toXML(XMLStreamWriter out) throws XMLStreamException {
+ out.writeCharacters(" ");
+ out.writeStartElement("group");
+ out.writeAttribute("name", group.getName());
+ out.writeAttribute("order", order);
+ if (locale != null) {
+ out.writeAttribute("locale", locale);
+ }
+ out.writeAttribute("maxtags", Integer.toString(maxTags));
+ out.writeAttribute("mincount", Integer.toString(minCount));
+ out.writeAttribute("offset", Integer.toString(offset));
+ out.writeAttribute("prefix", prefix);
+ out.writeCharacters("\n ");
+
+ out.writeStartElement("fields");
+ out.writeCharacters("\n");
+ for (ExposedRequest.Field field: group.getFields()) {
+ out.writeCharacters(" ");
+ out.writeStartElement("field");
+ out.writeAttribute("name", field.getField());
+ out.writeEndElement();
+ out.writeCharacters("\n");
+ }
+ out.writeCharacters(" ");
+ out.writeEndElement(); // fields
+ out.writeCharacters("\n ");
+ out.writeEndElement(); // group
+ out.writeCharacters("\n");
+ }
+
+ public String getOrder() {
+ return order;
+ }
+
+ public String getLocale() {
+ return locale;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public int getMaxTags() {
+ return maxTags;
+ }
+
+ public int getMinCount() {
+ return minCount;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+ }
+
+ /* Mutators */
+
+ public String getOrder() {
+ return order;
+ }
+
+ public void setOrder(String order) {
+ this.order = order;
+ }
+
+ public String getLocale() {
+ return locale;
+ }
+
+ public void setLocale(String locale) {
+ this.locale = locale;
+ }
+
+ public int getMaxTags() {
+ return maxTags;
+ }
+
+ public void setMaxTags(int maxTags) {
+ this.maxTags = maxTags;
+ }
+
+ public int getMinCount() {
+ return minCount;
+ }
+
+ public void setMinCount(int minCount) {
+ this.minCount = minCount;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public void setPrefix(String prefix) {
+ this.prefix = prefix;
+ }
+
+ public String getQuery() {
+ return stringQuery;
+ }
+
+ /**
+ * Warning: Changing the query is not recommended as it is used af key in
+ * cahce pools.
+ * @param query the new query.
+ */
+ public void setQuery(String query) {
+ this.stringQuery = query;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java Wed Sep 22 15:10:23 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPoolFactory.java Wed Sep 22 15:10:23 CEST 2010
@@ -0,0 +1,114 @@
+package org.apache.lucene.search.exposed.facet;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.exposed.ExposedCache;
+import org.apache.lucene.search.exposed.TermProvider;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Constructs {@link CollectorPool}s based on {@link FacetRequest}s. As the
+ * construction of a pool is costly, pools are kept in a LRU-cache.
+ *
+ * The factory connects to the {@link ExposedCache} and the cache is
+ * automatically updated (aka cleared) when the index is reloaded.
+ */
+public class CollectorPoolFactory implements ExposedCache.PurgeCallback {
+ private Map poolMap;
+ /**
+ * The maximum number of filled collectors to cache for each CollectorPool.
+ */
+ private final int filledCollectors;
+
+ /**
+ * The maximum number of fresh collectors to cache for each CollectorPool.
+ */
+ private final int freshCollectors;
+
+ private static CollectorPoolFactory lastFactory = null;
+
+ /**
+ * Constructing a CollectorPool is very costly, while constructing a collector
+ * in a collector pool only impacts garbage collections. In order to avoid
+ * excessive CPU-load, it is highly recommended to tweak maxSize,
+ * freshCollectors and filledCollectors so that maxSize is never reached.
+ *
+ * A new CollectorPool is constructed every time the group in the
+ * {@link FacetRequest} is modified. A new collector is constructed when
+ * a facet request is made and all current collectors for the given pool
+ * are active.
+ * @param maxSize the maximum number of CollectorPools to keep cached.
+ * @param filledCollectors the maximum number of fresh collectors in each
+ * pool.
+ * @param freshCollectors the maximum number of fresh collectors in each pool.
+ * @see {@link CollectorPool(FacetMap, int, int)}.
+ */
+ public CollectorPoolFactory(
+ final int maxSize, int filledCollectors, int freshCollectors) {
+ poolMap = new LinkedHashMap(maxSize, 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(
+ Map.Entry eldest) {
+ return size() > maxSize;
+ }
+ };
+ this.freshCollectors = freshCollectors;
+ this.filledCollectors = filledCollectors;
+ lastFactory = this;
+ }
+
+ /**
+ * An ugly hack to get singleton-like behaviour with initialization.
+ * @return the last factory generated.
+ */
+ // TODO: Replace this with proper singleton
+ public static CollectorPoolFactory getLastFactory() {
+ return lastFactory;
+ }
+
+ /**
+ * If a matching CollectorPool exists, it is returned. If not, a new pool is
+ * created. Note that pool creation is costly.
+ * @param reader a reader for the full index.
+ * @param request specified the behaviour of the wanted CollectorPool.
+ * @return a CollectorPool matching the request.
+ * @throws java.io.IOException if the reader could not be accessed.
+ */
+ public synchronized CollectorPool acquire(
+ IndexReader reader, FacetRequest request) throws IOException {
+ final String key = request.getGroupKey();
+ CollectorPool pool = poolMap.get(key);
+ if (pool == null) {
+ System.out.println("Creating CollectorPool for " + key);
+ List groups = request.getGroups();
+ List termProviders =
+ new ArrayList(groups.size());
+ for (FacetRequest.FacetGroup group: groups) {
+ termProviders.add(
+ ExposedCache.getInstance().getProvider(reader, group.getGroup()));
+ }
+
+ FacetMap facetMap = new FacetMap(reader.maxDoc(), termProviders);
+ pool = new CollectorPool(facetMap, filledCollectors, freshCollectors);
+ poolMap.put(key, pool);
+ }
+ return pool;
+ }
+
+ public synchronized void clear() {
+ poolMap.clear();
+ }
+
+ public void purgeAllCaches() {
+ clear();
+ }
+
+ public void purge(IndexReader r) {
+ // TODO: Make a selective clear
+ clear();
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java Sun Sep 19 20:12:28 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/CollectorPool.java Sun Sep 19 20:12:28 CEST 2010
@@ -0,0 +1,124 @@
+package org.apache.lucene.search.exposed.facet;
+
+import java.util.*;
+
+/**
+ * Holds a number of {@link TagCollector}s tied to a single {@link FacetMap}
+ * and ensures that they are cleared and ready for use. If there are no
+ * collectors in the pool when a new one is acquired, a new collector will be
+ * created.
+ *
+ * The pool holds a mix of cleared and ready for use {@link TagCollector}s as
+ * well as updated collectors from previous queries. When a collector is
+ * released back into the pool it is stored in the non-cleared cache. If the
+ * non-cleared cache is full, the oldest entry is cleared and moved into the
+ * cleared cache.
+ *
+ * The pool is thread-safe and thread-efficient.
+ */
+// TODO: Consider making a limit on the number of collectors to create
+public class CollectorPool {
+ private final List fresh;
+ private final Map filled;
+ private FacetMap map;
+ private final int freshCollectors;
+ private final int maxFilled;
+
+ /**
+ * When a TagCollector is released with a query, it is stored as filled. If
+ * the cache for filled is full, the oldest entry is cleared and stored in
+ * the fresh pool.
+ * @param map the map to use the collectors with.
+ * @param filledCollectors the maximum number of previously filled collectors.
+ * @param freshCollectors the maximum number of fresh collectors.
+ */
+ public CollectorPool(
+ FacetMap map, int filledCollectors, int freshCollectors) {
+ this.map = map;
+ this.freshCollectors = freshCollectors;
+ fresh = new ArrayList();
+ this.maxFilled = filledCollectors;
+ filled = new LinkedHashMap(filledCollectors) {
+ @Override
+ protected boolean removeEldestEntry(
+ Map.Entry eldest) {
+ if (size() > maxFilled) {
+ releaseFresh(remove(eldest.getKey())); // Send it to the fresh cache
+ }
+ return false; // We handle the removal ourselves
+ }
+ };
+ }
+
+ /**
+ * @param query the query associated with the collector. If the pool contains
+ * a collector filled from the query, it is returned. Else a cleared collector
+ * is returned. null is a valid query and will always return a cleared
+ * collector.
+ *
+ * Note: If a non-null query is provided, the caller must check the returned
+ * collector with the call {@link TagCollector#getQuery()}. If the result is
+ * not null, the collector was taken from cache and is already filled. In that
+ * case, the caller should not make a new search with the collector, but
+ * instead call {@link TagCollector#extractResult(FacetRequest)} directly.
+ * @return a recycled collector if one is available, else a new collector.
+ */
+ public TagCollector acquire(String query) {
+ synchronized (filled) {
+ if (query != null && maxFilled != 0 && filled.containsKey(query)) {
+ return filled.remove(query);
+ }
+ }
+
+ synchronized (fresh) {
+ for (int i = 0 ; i < fresh.size() ; i++) {
+ if (!fresh.get(i).isClearRunning()) {
+ return fresh.remove(i);
+ }
+ }
+ }
+ return new TagCollector(map);
+ }
+
+ /**
+ * Releasing a collector sends it to the filled cache. If the filled cache is
+ * full, the oldest entry is cleared with a delayedClear and sent to the fresh
+ * cache. If the maximum capacity of the fresh cache has been reached, the
+ * collector is discarded.
+ *
+ * This method exits immediately.
+ * @param query the query used for filling this collector. If null, the
+ * collector is always going to the fresh cache.
+ * @param collector the collector to put back into the pool for later reuse.
+ */
+ public void release(String query, TagCollector collector) {
+ synchronized (filled) {
+ if (maxFilled > 0 && query != null) {
+ collector.setQuery(query);
+ filled.put(query, collector);
+ return;
+ }
+ }
+ releaseFresh(collector);
+ }
+
+ private void releaseFresh(TagCollector collector) {
+ synchronized (fresh) {
+ if (fresh.size() >= freshCollectors) {
+ return;
+ }
+ collector.delayedClear();
+ fresh.add(collector);
+ }
+ }
+
+ public void clear() {
+ synchronized (fresh) {
+ fresh.clear();
+ }
+ synchronized (filled) {
+ filled.clear();
+ }
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java Thu Aug 12 13:57:51 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java Thu Aug 12 13:57:51 CEST 2010
@@ -0,0 +1,94 @@
+package org.apache.lucene.search.exposed.poc;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Analyzer for testing
+ */
+public final class MockAnalyzer extends Analyzer {
+ private final CharacterRunAutomaton runAutomaton;
+ private final boolean lowerCase;
+ private final CharacterRunAutomaton filter;
+ private final boolean enablePositionIncrements;
+
+ /**
+ * Creates a new MockAnalyzer.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ * @param filter DFA describing how terms should be filtered (set of stopwords, etc)
+ * @param enablePositionIncrements true if position increments should reflect filtered terms.
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ this.runAutomaton = runAutomaton;
+ this.lowerCase = lowerCase;
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ /**
+ * Creates a new MockAnalyzer, with no filtering.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false);
+ }
+
+ /**
+ * Create a Whitespace-lowercasing analyzer with no stopwords removal
+ */
+ public MockAnalyzer() {
+ this(MockTokenizer.WHITESPACE, true);
+ }
+
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ return new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
+ }
+
+ private class SavedStreams {
+ MockTokenizer tokenizer;
+ MockTokenFilter filter;
+ }
+
+ @Override
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
+ SavedStreams saved = (SavedStreams) getPreviousTokenStream();
+ if (saved == null) {
+ saved = new SavedStreams();
+ saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
+ setPreviousTokenStream(saved);
+ return saved.filter;
+ } else {
+ saved.tokenizer.reset(reader);
+ return saved.filter;
+ }
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java Fri Sep 17 21:06:04 CEST 2010
+++ lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java Fri Sep 17 21:06:04 CEST 2010
@@ -0,0 +1,25 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Returns the index as the value in {@link #get}. Useful for representing a 1:1
+ * mapping of another structure.
+ */
+public class IdentityReader implements PackedInts.Reader {
+ private int size;
+
+ public IdentityReader(int size) {
+ this.size = size;
+ }
+
+ public long get(int index) {
+ return index;
+ }
+
+ public int getBitsPerValue() {
+ return 0;
+ }
+
+ public int size() {
+ return size;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java Thu Sep 23 11:39:33 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java Thu Sep 23 11:39:33 CEST 2010
@@ -0,0 +1,190 @@
+/* $Id:$
+ *
+ * The Summa project.
+ * Copyright (C) 2005-2010 The State and University Library
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+/**
+ * Custom sorter that uses the Exposed framework. Trade-offs are slow first-time
+ * use, fast subsequent use and low memory footprint.
+ */
+public class ExposedFieldComparatorSource extends FieldComparatorSource {
+ private final IndexReader reader;
+ private final Comparator comparator;
+ private final String comparatorID;
+ private final boolean sortNullFirst;
+
+ // TODO: Figure out how to avoid the need for re-creation upon re-open
+ public ExposedFieldComparatorSource(
+ IndexReader reader, Comparator comparator, String comparatorID,
+ boolean sortNullFirst) {
+ this.reader = reader;
+ this.comparatorID = comparatorID;
+ this.comparator = comparator;
+ this.sortNullFirst = sortNullFirst;
+ }
+
+ public ExposedFieldComparatorSource(IndexReader reader, Locale locale) {
+ this(reader,
+ ExposedComparators.collatorToBytesRef(
+ locale == null ? null : Collator.getInstance(locale)),
+ locale == null ? ExposedRequest.LUCENE_ORDER : locale.toString(),
+ false);
+ }
+
+ /**
+ * @param fieldname the field to sort on. If a group of fields is wanted,
+ * field names must be separated by {@code ;}.
+ * Example: {@code foo;bar;zoo}. Note that grouping treats the terms
+ * from the separate fields at the same level and that there should
+ * only be a single term i total for the given fields for a single
+ * document. Grouping is not normally used at this level.
+ * @param numHits the number of hits to return.
+ * @param sortPos ?
+ * @param reversed reversed sort order.
+ * @return a comparator based on the given parameters.
+ * @throws IOException if the
+ */
+ @Override
+ public FieldComparator newComparator(
+ String fieldname, int numHits, int sortPos, boolean reversed)
+ throws IOException {
+ List fieldNames = Arrays.asList(fieldname.split(":"));
+ return new ExposedFieldComparator(
+ reader, fieldname, fieldNames, numHits, sortPos, reversed);
+ }
+
+ private class ExposedFieldComparator extends FieldComparator {
+ private final TermProvider provider;
+
+ private final int undefinedTerm = -1;
+ private final PackedInts.Reader docOrder;
+
+ private final String groupName;
+ private final List fieldNames;
+ private final int numHits;
+ private final int sortPos;
+ private final boolean reversed;
+ private final int factor; // Reverse-indicator
+ private int docBase = 0; // Added to all incoming docIDs
+
+ private int[] order; // docOrder
+ private int bottom; // docOrder
+
+ public ExposedFieldComparator(
+ IndexReader reader, String groupName, List fieldNames,
+ int numHits, int sortPos, boolean reversed) throws IOException {
+ this.groupName = groupName;
+ this.fieldNames = fieldNames;
+ this.provider = ExposedCache.getInstance().getProvider(
+ reader, groupName, fieldNames, comparator, comparatorID);
+
+ this.numHits = numHits;
+ this.sortPos = sortPos;
+ this.reversed = reversed;
+ factor = reversed ? -1 : 1;
+ docOrder = provider.getDocToSingleIndirect();
+
+ order = new int[numHits];
+ }
+
+ @Override
+ public int compare(int slot1, int slot2) {
+ if (!sortNullFirst) {
+ int slot1order = order[slot1];
+ int slot2order = order[slot2];
+ if (slot1order == undefinedTerm) {
+ return slot2order == undefinedTerm ? 0 : -1;
+ } else if (slot2order == undefinedTerm) {
+ return 1;
+ }
+ return factor * (slot1order - slot2order);
+ }
+ // No check for null as null-values are always assigned -1
+ return factor * (order[slot1] - order[slot2]);
+ }
+
+ @Override
+ public void setBottom(int slot) {
+ bottom = order[slot];
+ }
+
+ @Override
+ public int compareBottom(int doc) throws IOException {
+ if (!sortNullFirst) {
+ final long bottomOrder = bottom;
+ long docOrderR = docOrder.get(doc+docBase);
+ if (bottomOrder == undefinedTerm) {
+ return docOrderR == undefinedTerm ? 0 : -1;
+ } else if (docOrderR == undefinedTerm) {
+ return 1;
+ }
+ return (int)(factor * (bottomOrder - docOrderR));
+ }
+ // No check for null as null-values are always assigned -1
+ return (int)(factor * (bottom - docOrder.get(doc+docBase)));
+ }
+
+ @Override
+ public void copy(int slot, int doc) throws IOException {
+ // TODO: Remove this
+// System.out.println("Copy called: order[" + slot + "] = "
+// + doc + "+" + docBase + " = " + (doc + docBase));
+// System.out.println("docID " + (doc+docBase) + " has term " + provider.getOrderedTerm(doc+docBase).utf8ToString());
+ order[slot] = (int)docOrder.get(doc+docBase);
+ }
+
+ @Override
+ public FieldComparator setNextReader(IndexReader reader, int docBase)
+ throws IOException {
+ this.docBase = docBase;
+ return this;
+ }
+
+ private final BytesRef EMPTY = new BytesRef("");
+ @Override
+ public Comparable value(int slot) {
+ try { // A bit cryptic as we need to handle the case of no sort term
+ final long resolvedDocOrder = order[slot];
+ // TODO: Remove this
+/* System.out.println("Resolving docID " + slot + " with docOrder entry "
+ + resolvedDocOrder + " to term "
+ + (resolvedDocOrder == undefinedTerm ? "null" :reader.getTermText(
+ (int)termOrder.get((int)resolvedDocOrder))));
+ */
+ return resolvedDocOrder == undefinedTerm ? EMPTY :
+ provider.getOrderedTerm(resolvedDocOrder);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while extracting term String", e);
+ }
+ }
+ }
+
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xml
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xml Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xml Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,31 @@
+
+
+
+ freetext:"foo <> bar & zoo"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,251 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.Comparator;
+
+/**
+ * Wraps different types of comparators.
+ */
+public class ExposedComparators {
+
+ /**
+ * Simple atomic specialization of Comparator. Normally used to
+ * compare terms by treating the arguments as ordinals and performing lookup
+ * in an underlying reader.
+ */
+ public interface OrdinalComparator {
+ int compare(int value1, int value2);
+ }
+
+ public static Comparator collatorToBytesRef(
+ final Collator collator) {
+ return collator == null ? new NaturalComparator()
+ : new BytesRefWrappedCollator(collator);
+ }
+ public static final class BytesRefWrappedCollator implements
+ Comparator {
+ private final Collator collator;
+ public BytesRefWrappedCollator(Collator collator) {
+ this.collator = collator;
+ }
+ public int compare(BytesRef o1, BytesRef o2) {
+ return collator.compare(o1.utf8ToString(), o2.utf8ToString());
+ }
+ public Collator getCollator() {
+ return collator;
+ }
+ }
+ public static final class NaturalComparator implements Comparator {
+ public int compare(BytesRef o1, BytesRef o2) {
+ return o1.compareTo(o2); // TODO: Consider null-case
+ }
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the BytesRefs values and feeding them to the comparator.
+ */
+ public static OrdinalComparator wrap(TermProvider provider,
+ Comparator comparator) {
+ return new BytesRefWrapper(provider, comparator);
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param map indirect map from logical index to term.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the BytesRefs values and feeding them to the comparator.
+ * The BytesRefs are requested by calling getTerm with map[value].
+ */
+ public static OrdinalComparator wrapIndirect(
+ TermProvider provider, int[] map, Comparator comparator) {
+ return new IndirectBytesRefWrapper(provider, map, comparator);
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param map indirect map from logical index to ordinal.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the CollatorKeys and comparing them.
+ * The keys are requested by calling get with map[value].
+ */
+ public static OrdinalComparator wrapIndirect(
+ CachedCollatorKeyProvider provider, int[] map) {
+ return new IndirectCollatorKeyWrapper(provider, map);
+ }
+
+ /**
+ * Creates a comparator that relies on a list of ExposedTuples to get the
+ * BytesRef for comparison.
+ * @param backingTuples the tuples with the terms to use for comparisons.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares indexes relative to the backingTuples.
+ * The BytesRefs are requested by calling {@code backingTuples[index].term}.
+ */
+ public static OrdinalComparator wrapBacking(
+ ExposedTuple[] backingTuples, Comparator comparator) {
+ return new BackingTupleWrapper(backingTuples, comparator);
+ }
+
+ public static OrdinalComparator wrapBacking(CollationKey[] backingKeys) {
+ return new BackingCollatorWrapper(backingKeys);
+ }
+
+ /**
+ * Compares the given ordinals directly.
+ */
+ public static final class DirectComparator implements OrdinalComparator {
+ public final int compare(final int value1, final int value2) {
+ return value1-value2;
+ }
+ }
+
+ /**
+ * Uses the given values as indexes in a backing int[], comparing the values
+ * from the int[] directly.
+ */
+ public static final class IndirectComparator implements OrdinalComparator {
+ private final int[] values;
+
+ public IndirectComparator(int[] values) {
+ this.values = values;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ return values[value1]-values[value2];
+ }
+ }
+
+ /**
+ * Like {@link IndirectComparator} but in reverse order.
+ */
+ public static final class ReverseIndirectComparator
+ implements OrdinalComparator {
+ private final int[] values;
+
+ public ReverseIndirectComparator(int[] values) {
+ this.values = values;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ return values[value2]-values[value1];
+ }
+ }
+
+ private static class BytesRefWrapper implements OrdinalComparator {
+ private final TermProvider provider;
+ private final Comparator comparator;
+
+ public BytesRefWrapper(
+ TermProvider provider, Comparator comparator) {
+ this.comparator = comparator;
+ this.provider = provider;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ try {
+ return comparator == null ?
+ provider.getTerm(value1).compareTo(provider.getTerm(value2)) :
+ comparator.compare(
+ provider.getTerm(value1), provider.getTerm(value2));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing terms for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class IndirectBytesRefWrapper implements OrdinalComparator {
+ private final TermProvider provider;
+ private final int[] map;
+ private final Comparator comparator;
+
+ public IndirectBytesRefWrapper(
+ TermProvider provider, int[] map, Comparator comparator) {
+ this.provider = provider;
+ this.map = map;
+ this.comparator = comparator;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ try {
+ return comparator.compare(
+ provider.getTerm(map[value1]), provider.getTerm(map[value2]));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing terms for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class IndirectCollatorKeyWrapper implements OrdinalComparator {
+ private final CachedCollatorKeyProvider provider;
+ private final int[] map;
+
+ public IndirectCollatorKeyWrapper(
+ CachedCollatorKeyProvider provider, int[] map) {
+ this.provider = provider;
+ this.map = map;
+ }
+
+ // TODO: Consider null as value
+ public final int compare(final int value1, final int value2) {
+ try {
+ return provider.get(map[value1]).compareTo(provider.get(map[value2]));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing keys for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class BackingTupleWrapper implements OrdinalComparator {
+ public ExposedTuple[] backingTerms; // Updated from the outside
+ private final Comparator comparator;
+
+ private BackingTupleWrapper(
+ ExposedTuple[] backingTerms, Comparator comparator) {
+ this.backingTerms = backingTerms;
+ this.comparator = comparator;
+ }
+
+ public int compare(int v1, int v2) {
+ try {
+ return comparator == null ?
+ backingTerms[v1].term.compareTo(backingTerms[v2].term) :
+ comparator.compare(backingTerms[v1].term, backingTerms[v2].term);
+ } catch (NullPointerException e) {
+ System.out.println("ddd");
+ return 0;
+ }
+ }
+ }
+
+ private static class BackingCollatorWrapper implements OrdinalComparator {
+ public CollationKey[] backingKeys; // Updated from the outside
+
+ public BackingCollatorWrapper(CollationKey[] backingKeys) {
+ this.backingKeys = backingKeys;
+ }
+
+ public int compare(int v1, int v2) {
+ try {
+ return backingKeys[v1].compareTo(backingKeys[v2]);
+ } catch (NullPointerException e) {
+ return 0;
+ }
+ }
+
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java Sat Sep 18 01:29:36 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java Sat Sep 18 01:29:36 CEST 2010
@@ -0,0 +1,141 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+
+/**
+ * Wrapper for TermProvider that provides a flexible cache for terms.
+ * Warning: This cache does not work well for docIDs.
+ *
+ * For random access, a plain setup with no read-ahead is suitable.
+ *
+ * For straight iteration in ordinal order, a setup with maximum read-ahead
+ * is preferable. This is uncommon.
+ *
+ * For iteration in sorted ordinal order, where the ordinal order is fairly
+ * aligned to unicode sorting order, a setup with some read-ahead works well.
+ * This is a common case.
+ *
+ * For merging chunks (See the SegmentReader.getSortedTermOrdinals), where
+ * the ordinal order inside the chunks if fairly aligned to unicode sorting
+ * order, a read-ahead works iff {@link #onlyReadAheadIfSpace} is true as this
+ * prevents values from the beginning of other chunks from being flushed when
+ * values from the current chunk is filled with read-ahead. This requires that
+ * the merger removes processed values from the cache explicitly.
+ *
+ * If the underlying order is used by calling {@link #getOrderedTerm(long)},
+ * {@link #getOrderedField(long)} or {@link #getOrderedOrdinals()}, the
+ * methods are delegated directly to the backing TermProvider. It is the
+ * responsibility of the caller to endure that proper caching of the order
+ * is done at the source level.
+ */
+public class CachedTermProvider extends CachedProvider
+ implements TermProvider {
+ private TermProvider source;
+
+ /**
+ *
+ * @param source the backing term provider.
+ * @param cacheSize the maximum number of elements to hold in cache.
+ * @param readAhead the maximum number of lookups that can be performed
+ * after a plain lookup.
+ * @throws java.io.IOException if the cache could access the source.
+ */
+ public CachedTermProvider(
+ TermProvider source, int cacheSize, int readAhead) throws IOException {
+ super(cacheSize, readAhead, source.getOrdinalTermCount()-1);
+ this.source = source;
+ }
+
+ @Override
+ protected BytesRef lookup(final long index) throws IOException {
+ return source.getTerm(index);
+ }
+
+ public BytesRef getTerm(final long ordinal) throws IOException {
+ return get(ordinal);
+ }
+
+ public Iterator getIterator(boolean collectDocIDs)
+ throws IOException {
+ throw new UnsupportedOperationException(
+ "The cache does not support the creation of iterators");
+ }
+
+ public String getDesignation() {
+ return "CachedTermProvider(" + source.getDesignation() + ")";
+ }
+
+ /* Delegations */
+
+ public int getNearestTermIndirect(BytesRef key) throws IOException {
+ return source.getNearestTermIndirect(key);
+ }
+
+ public int getNearestTermIndirect(BytesRef key, int startTermPos, int endTermPos) throws IOException {
+ return source.getNearestTermIndirect(key, startTermPos, endTermPos);
+ }
+
+ public Comparator getComparator() {
+ return source.getComparator();
+ }
+
+ public String getComparatorID() {
+ return source.getComparatorID();
+ }
+
+ public String getField(long ordinal) throws IOException {
+ return source.getField(ordinal);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return source.getOrderedField(indirect);
+ }
+
+ public BytesRef getOrderedTerm(long indirect) throws IOException {
+ return source.getOrderedTerm(indirect);
+ }
+
+ public long getUniqueTermCount() throws IOException {
+ return source.getUniqueTermCount();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return source.getOrdinalTermCount();
+ }
+
+ public long getMaxDoc() {
+ return source.getMaxDoc();
+ }
+
+ public IndexReader getReader() throws IOException {
+ return source.getReader();
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ return source.getDocsEnum(ordinal, reuse);
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ return source.getOrderedOrdinals();
+ }
+
+ public PackedInts.Reader getDocToSingleIndirect() throws IOException {
+ return source.getDocToSingleIndirect();
+ }
+
+ public int getReaderHash() {
+ return source.getReaderHash();
+ }
+
+ public int getRecursiveHash() {
+ return source.getRecursiveHash();
+ }
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,274 @@
+package org.apache.lucene.search.exposed.poc;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.exposed.ExposedFieldComparatorSource;
+import org.apache.lucene.search.exposed.ExposedSettings;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Locale;
+
+public class ExposedPOC {
+ private static final double MAX_HITS = 20;
+
+ public static void main(String[] args)
+ throws IOException, InterruptedException, ParseException {
+ if (args.length == 2 && "optimize".equals(args[0])) {
+ optimize(new File(args[1]));
+ return;
+ }
+ if (args.length != 6) {
+ System.err.println("Need 6 arguments, got " + args.length + "\n");
+ usage();
+ return;
+ }
+ String method = args[0];
+ File location = new File(args[1]);
+ String field = args[2];
+ Locale locale =
+ "null".equalsIgnoreCase(args[3]) ? null : new Locale(args[3]);
+ String defaultField = args[4];
+ ExposedSettings.PRIORITY priority =
+ ExposedSettings.PRIORITY.fromString(args[5]);
+ try {
+ shell(method, location, field, locale, defaultField, priority);
+ } catch (Exception e) {
+ //noinspection CallToPrintStackTrace
+ e.printStackTrace();
+ usage();
+ }
+ }
+
+ private static void optimize(File location) throws IOException {
+ System.out.println("Optimizing " + location + "...");
+ long startTimeOptimize = System.nanoTime();
+ IndexWriter writer = new IndexWriter(FSDirectory.open(location),
+ new IndexWriterConfig(Version.LUCENE_31, new MockAnalyzer()));
+ writer.optimize(1);
+ System.out.println("Optimized index in " + nsToString(
+ System.nanoTime() - startTimeOptimize));
+ writer.close(true);
+ }
+
+ private static void shell(
+ String method, File location, String field, Locale locale,
+ String defaultField, ExposedSettings.PRIORITY priority)
+ throws IOException, InterruptedException, ParseException {
+ System.out.println(String.format(
+ "Testing sorted search for index at '%s' with sort on field %s with " +
+ "locale %s, using sort-method %s, priority %s. Heap: %s",
+ location, field, locale, method, priority, getHeap()));
+ ExposedSettings.priority = priority;
+
+ IndexReader reader = IndexReader.open(FSDirectory.open(location), true);
+ System.out.println(String.format(
+ "Opened index of size %s from %s. the indes has %d documents and %s" +
+ " deletions. Heap: %s",
+ readableSize(calculateSize(location)), location,
+ reader.maxDoc(),
+ reader.hasDeletions() ? "some" : " no",
+ getHeap()));
+
+/* System.out.println(String.format(
+ "Creating %s Sort for field %s with locale %s... Heap: %s",
+ method, field, locale, getHeap()));
+ */
+ long startTimeSort = System.nanoTime();
+ Sort sort;
+ if ("exposed".equals(method) || "expose".equals(method)) {
+ ExposedFieldComparatorSource exposedFCS =
+ new ExposedFieldComparatorSource(reader, locale);
+ sort = new Sort(new SortField(field, exposedFCS));
+ } else if ("default".equals(method)) {
+ sort = locale == null ?
+ new Sort(new SortField(field, SortField.STRING)) :
+ new Sort(new SortField(field, locale));
+ } else {
+ throw new IllegalArgumentException(
+ "The sort method " + method + " is unsupported");
+ }
+ long sortTime = System.nanoTime() - startTimeSort;
+
+ System.out.println(String.format(
+ "Created %s Sort for field %s in %s. Heap: %s",
+ method, field, nsToString(sortTime), getHeap()));
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ System.out.println(String.format(
+ "\nFinished initializing %s structures for field %s.\n"
+ + "Write standard Lucene queries to experiment with sorting speed.\n"
+ + "The StandardAnalyser will be used and the default field is %s.\n"
+ + "Finish with 'EXIT'.", method, field, defaultField));
+ String query;
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, defaultField, new MockAnalyzer());
+
+ boolean first = true;
+ while (true) {
+ if (first) {
+ System.out.print("\nQuery (" + method + " sort, first search might take " +
+ "a while): ");
+ first = false;
+ } else {
+ System.out.print("Query (" + method + " sort): ");
+ }
+ query = br.readLine();
+ if ("".equals(query)) {
+ continue;
+ }
+ if ("EXIT".equals(query)) {
+ break;
+ }
+ try {
+ long startTimeQuery = System.nanoTime();
+ Query q = qp.parse(query);
+ long queryTime = System.nanoTime() - startTimeQuery;
+
+ long startTimeSearch = System.nanoTime();
+ TopFieldDocs topDocs = searcher.search(
+ q.weight(searcher), null, 20, sort, true);
+ long searchTime = System.nanoTime() - startTimeSearch;
+ System.out.println(String.format(
+ "The search for '%s' got %d hits in %s (+ %s for query parsing). "
+ + "Showing %d hits.",
+ query, topDocs.totalHits,
+ nsToString(searchTime),
+ nsToString(queryTime),
+ (int)Math.min(topDocs.totalHits, MAX_HITS)));
+ long startTimeDisplay = System.nanoTime();
+ for (int i = 0 ; i < Math.min(topDocs.totalHits, MAX_HITS) ; i++) {
+ int docID = topDocs.scoreDocs[i].doc;
+ System.out.println(String.format(
+ "Hit #%d was doc #%d with %s:%s",
+ i, docID, field,
+ ((BytesRef)((FieldDoc)topDocs.scoreDocs[i]).fields[0]).
+ utf8ToString()));
+ }
+ System.out.print("Displaying the search result took "
+ + nsToString(
+ System.nanoTime() - startTimeDisplay) + ". ");
+ System.out.println("Heap: " + getHeap());
+ } catch (Exception e) {
+ //noinspection CallToPrintStackTrace
+ e.printStackTrace();
+ }
+ }
+ System.out.println("\nThank you for playing. Please come back.");
+ }
+
+ private static void usage() {
+ System.out.println(
+ "Usage: ExposedPOC exposed|default " +
+ " \n"
+ + "exposed: Uses the expose sorter\n"
+ + "default: Uses the default sorter\n"
+ + ": The location of an optimized Lucene index\n"
+ + ": The field to use for sorting\n"
+ + ": The locale to use for sorting. If null is "
+ + "specified, natural term order is used\n"
+ + ": The field to search when no explicit field is " +
+ "given\n"
+ + ": Either speed or memory. Only relevant for " +
+ "exposed\n"
+ + "\n"
+ + "Example:\n"
+ + "ExposedPOC expose /mnt/bulk/40GB_index author da freetext"
+ + "\n"
+ + "If the index is is to be optimized, it can be done with\n"
+ + "ExposedPOC optimize \n"
+ + "\n"
+ + "Note: Heap-size is queried after a call to System.gc()\n"
+ + "Note: This is a proof of concept. Expect glitches!"
+ );
+ }
+
+ static String getHeap() throws InterruptedException {
+ String b = "Before GC: " + getHeapDirect();
+ for (int i = 0 ; i < 1 ; i++) {
+ System.gc();
+ Thread.sleep(10);
+ }
+ return b + ", after GC: " + getHeapDirect();
+ }
+
+ private static String getHeapDirect() {
+ return readableSize(Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory()) + "/"
+ + readableSize(Runtime.getRuntime().totalMemory());
+ }
+
+ static String readableSize(long size) {
+ return size > 2 * 1048576 ?
+ size / 1048576 + "MB" :
+ size > 2 * 1024 ?
+ size / 1024 + "KB" :
+ size + "bytes";
+ }
+
+ static long calculateSize(File file) {
+ long size = 0;
+ if (file.isDirectory()) {
+ for (File sub: file.listFiles()) {
+ size += calculateSize(sub);
+ }
+ } else {
+ size += file.length();
+ }
+ return size;
+ }
+
+ static String measureSortTime(final PackedInts.Reader orderedDocs) {
+ Integer[] allDocIDS = new Integer[orderedDocs.size()];
+ for (int i = 0 ; i < allDocIDS.length ; i++) {
+ allDocIDS[i] = i;
+ }
+ long startTimeSort = System.nanoTime();
+ Arrays.sort(allDocIDS, new Comparator() {
+ public int compare(Integer o1, Integer o2) {
+ return (int)(orderedDocs.get(o1) - orderedDocs.get(o2));
+ }
+ });
+ return nsToString(
+ System.nanoTime() - startTimeSort);
+ }
+
+ static long footprint(PackedInts.Reader values) {
+ return values.getBitsPerValue() * values.size() / 8;
+ }
+
+ static String nsToString(long time) {
+ return time > 10L * 1000 * 1000000 ?
+ minutes(time) + " min" :
+// time > 2 * 1000000 ?
+ time / 1000000 + "ms";// :
+// time + "ns";
+ }
+
+ private static String minutes(long num) {
+ long min = num / 60 / 1000 / 1000000;
+ long left = num - (min * 60 * 1000 * 1000000);
+ long sec = left / 1000 / 1000000;
+ String s = Long.toString(sec);
+ while (s.length() < 2) {
+ s = "0" + s;
+ }
+ return min + ":" + s;
+ }
+
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xsd
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xsd Sun Sep 19 13:08:46 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetRequest.xsd Sun Sep 19 13:08:46 CEST 2010
@@ -0,0 +1,95 @@
+
+
+
+
+ XML Schema for Lucene exposed (LUCENE-2369) requests.
+
+
+
+
+
+
+
+
+ The query is the unmodified query used for the Lucene search to
+ provide faceting for.
+ order: count = the tags are sorted by occurrences.
+ Highest number comes first.
+ index = the tags sorted by index order.
+ locale = the tags are sorted by the locale given in the
+ attribute "locale".
+ The default order is count.
+ locale: If sort is specified to "locale", this locale is used.
+ maxtags: The maximum number of tags to return for a facet.
+ -1 is unlimited and _not_ recommended.
+ The default is 20.
+ mincount: The minimum number of occurrences in order for a tag
+ to be part of the result.
+ The default is 0.
+ offset: Where to start extracting tags in the sorted list of
+ tags. Used for pagination.
+ The default is 0.
+ Note: This can be negative when used with "prefix" and
+ order != count.
+ prefix: The extraction starts at (the first tag that matches the
+ prefix) + offset. This cannot be used with count order.
+ The default is "", meaning the beginning of the sorted
+ tag list.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,81 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+public class ExposedFactory {
+
+ // TODO: Split this so that group is explicit
+ public static TermProvider createProvider(
+ IndexReader reader, String groupName, List fieldNames,
+ Comparator comparator, String comparatorID) throws IOException {
+ if (fieldNames.size() == 0) {
+ throw new IllegalArgumentException("There must be at least 1 field name");
+ }
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() == 1) {
+ // Segment-level single field
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ fieldNames.get(0), comparator, comparatorID);
+ return new FieldTermProvider(reader, request, true);
+ }
+
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() >= 1) {
+ // Segment-level multi field
+ List fieldProviders =
+ new ArrayList(fieldNames.size());
+ List fieldRequests =
+ new ArrayList(fieldNames.size());
+ for (String fieldName: fieldNames) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldName, comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(reader, fieldRequest, false));
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() == 1) {
+ // Index-level single field
+ IndexReader[] subs = reader.getSequentialSubReaders();
+ List fieldProviders =
+ new ArrayList(subs.length);
+ List fieldRequests =
+ new ArrayList(subs.length);
+ for (IndexReader sub: subs) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldNames.get(0), comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(sub, fieldRequest, false));
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+ // Index-level multi field
+ IndexReader[] subs = reader.getSequentialSubReaders();
+ List fieldProviders =
+ new ArrayList(subs.length * fieldNames.size());
+ List fieldRequests =
+ new ArrayList(subs.length * fieldNames.size());
+ for (IndexReader sub: subs) {
+ for (String fieldName: fieldNames) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldName, comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(sub, fieldRequest, false));
+ }
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java Sun Sep 19 21:31:42 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/TagCollector.java Sun Sep 19 21:31:42 CEST 2010
@@ -0,0 +1,348 @@
+package org.apache.lucene.search.exposed.facet;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.exposed.ExposedComparators;
+import org.apache.lucene.search.exposed.ExposedPriorityQueue;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Counts tag occurrences in the given documents. This collector can be used as
+ * a standard Lucene collector or it can be filled by specifying the document
+ * IDs with an {@link OpenBitSet} or a standard {@code int[]}}.
+ */
+public class TagCollector extends Collector {
+ private String query = null;
+ private long countTime = -1;
+ private int docBase;
+ private final int[] tagCounts;
+ private final FacetMap map;
+ private boolean clearRunning = false;
+ private long hitCount = 0;
+// TODO: Rememver query for caching of results
+ public TagCollector(FacetMap map) {
+ this.map = map;
+ this.tagCounts = new int[map.getTagCount()];
+ }
+
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ //Ignore
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ hitCount++;
+ map.updateCounter(tagCounts, doc + docBase);
+ }
+
+ @Override
+ public void setNextReader(IndexReader reader, int docBase) throws IOException {
+ this.docBase = docBase;
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return true;
+ }
+
+ /**
+ * Uses the given bits to update the tag counts. Each bit designates a docID.
+ *
+ * Note: This is different from calling {@link #collect(int)} repeatedly
+ * as the single docID collect method adjusts for docBase given in
+ * {@link #setNextReader}.
+ * @param bits the document IDs to use to use for tag counting.
+ * @throws java.io.IOException if the bits could not be accessed.
+ */
+ public void collect(OpenBitSet bits) throws IOException {
+ countTime = System.currentTimeMillis();
+ hitCount = 0;
+ DocIdSetIterator ids = bits.iterator();
+ int id;
+ while ((id = ids.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ hitCount++;
+ map.updateCounter(tagCounts, id);
+ }
+ countTime = System.currentTimeMillis() - countTime;
+ }
+
+ /**
+ * Uses the given docIDs to update the tag counts.
+ *
+ * Note: This is different from calling {@link #collect(int)} repeatedly
+ * as the single docID collect method adjusts for docBase given in
+ * {@link #setNextReader}.
+ * @param docIDs the document IDs to use for tag counting.
+ */
+ public void collect(int[] docIDs) {
+ countTime = System.currentTimeMillis();
+ hitCount = docIDs.length;
+ for (int docID: docIDs) {
+ map.updateCounter(tagCounts, docID);
+ }
+ countTime = System.currentTimeMillis() - countTime;
+ }
+
+ /**
+ * Collect all docIDs except those marked as deleted.
+ * @param reader a reader for the index.
+ * @throws java.io.IOException if the bits could not be accessed.
+ */
+ // TODO: Rewrite this to avoid allocation of the large bitset
+ public void collectAllValid(IndexReader reader) throws IOException {
+ countTime = System.currentTimeMillis();
+ OpenBitSet bits = new OpenBitSet(reader.maxDoc());
+ bits.flip(0, reader.maxDoc());
+ if (reader.getSequentialSubReaders() == null) {
+ remove(bits, 0, reader.getDeletedDocs());
+ } else {
+ for (IndexReader sub: reader.getSequentialSubReaders()) {
+ remove(bits, reader.getSubReaderDocBase(sub), sub.getDeletedDocs());
+ }
+ }
+ collect(bits);
+ countTime = System.currentTimeMillis() - countTime;
+ }
+ private void remove(OpenBitSet prime, int base, Bits removers) {
+ if (removers == null) {
+ return;
+ }
+ // TODO: Optimize for OpenBitSet
+ for (int i = 0 ; i < removers.length() ; i++) {
+ if (removers.get(i)) {
+ prime.clear(base + i);
+ }
+ }
+ }
+
+ /**
+ * Clears the collector making it ready for reuse. It is highly recommended to
+ * reuse collectors as it lowers Garbage Collection impact, especially for
+ * large scale faceting.
+ *
+ * Consider using {@link #delayedClear()} to improve responsiveness.
+ */
+ public void clear() {
+ clearRunning = true;
+ Arrays.fill(tagCounts, 0);
+ hitCount = 0;
+ query = null;
+ clearRunning = false;
+ }
+
+ /**
+ * Starts a Thread that clears the collector and exits immediately. Make sure
+ * that {@link #isClearRunning()} returns false before using the collector
+ * again.
+ */
+ public void delayedClear() {
+ new Thread(new Runnable() {
+ public void run() {
+ clear();
+ }
+ }, "TagCollector clear").start();
+ }
+
+ public boolean isClearRunning() {
+ return clearRunning;
+ }
+
+
+ public String toString() {
+ return "TagCollector(" + tagCounts.length + " potential tags from "
+ + map.toString();
+ }
+
+
+ /**
+ * After collection (and implicit reference counting), a response can be
+ * extracted. Note that countTime and totalTime is not set in the response.
+ * It is recommended that these two time measurements are updated before
+ * the response is delivered.
+ * @param request a request matching the one used for counting. Note that
+ * minor details, such as offset and maxTags need not match.
+ * @return a fully resolved facet structure.
+ * @throws IOException if the facet mapper could not deliver terms.
+ */
+ public FacetResponse extractResult(FacetRequest request) throws IOException {
+ if (map.getIndirectStarts().length-1 != request.getGroups().size()) {
+ throw new IllegalStateException(
+ "The number of term providers in the FacetMap was "
+ + (map.getIndirectStarts().length-1)
+ + ", while the number of groups in the request was "
+ + request.getGroups().size() + ". The two numbers must match");
+ }
+ List responseGroups =
+ new ArrayList(request.getGroups().size());
+ // TODO: Consider threading larger facets, but beware of cache blowout
+ for (int i = 0 ; i < request.getGroups().size() ; i++) {
+ FacetRequest.FacetGroup requestGroup = request.getGroups().get(i);
+// System.out.println("Extracting for " + requestGroup.getGroup().getName() + ": " + startTermPos + " -> " + endTermPos);
+ responseGroups.add(extractResult(requestGroup, i));
+ }
+ FacetResponse response =
+ new FacetResponse(request, responseGroups, hitCount);
+ response.setCountingTime(getCountTime());
+ return response;
+ }
+
+ private FacetResponse.Group extractResult(
+ FacetRequest.FacetGroup requestGroup, int groupID) throws IOException {
+/* if (!FacetRequest.ORDER_COUNT.equals(requestGroup.getOrder())) {
+ throw new UnsupportedOperationException("The order '"
+ + requestGroup.getOrder() + " is not supported yet for result " +
+ "extraction");
+ }
+ */
+ int startTermPos = map.getIndirectStarts()[groupID]; // Inclusive
+ int endTermPos = map.getIndirectStarts()[groupID+1]; // Exclusive
+ String order = requestGroup.getOrder();
+ if (FacetRequest.ORDER_COUNT.equals(order)) {
+ return extractCountResult(requestGroup, startTermPos, endTermPos);
+ } else if (FacetRequest.ORDER_INDEX.equals(order)
+ || FacetRequest.ORDER_LOCALE.equals(order)) {
+ return extractOrderResult(
+ requestGroup, groupID, startTermPos, endTermPos);
+ }
+ throw new UnsupportedOperationException(
+ "The order '" + order + "' is unknown");
+ }
+
+ private FacetResponse.Group extractOrderResult(
+ FacetRequest.FacetGroup requestGroup, final int groupID,
+ final int startTermPos, final int endTermPos) throws IOException {
+ long extractionTime = System.currentTimeMillis();
+ // Locate prefix
+ int origo = startTermPos;
+ if (requestGroup.getPrefix() != null
+ && !"".equals(requestGroup.getPrefix())) {
+ int tpOrigo = map.getProviders().get(groupID).getNearestTermIndirect(
+ new BytesRef(requestGroup.getPrefix()));
+ origo = tpOrigo + map.getIndirectStarts()[groupID];
+ //origo = origo < 0 ? (origo + 1) * -1 : origo; // Ignore missing match
+ }
+
+ // skip to offset (only valid)
+ final int minCount = requestGroup.getMinCount();
+ final int direction = requestGroup.getOffset() < 0 ? -1 : 1;
+ int delta = Math.abs(requestGroup.getOffset());
+ while (delta > 0) {
+ origo += direction;
+ if (origo < startTermPos || origo >= endTermPos) {
+ origo += delta-1;
+ break;
+ }
+ if (tagCounts[origo] >= minCount) {
+ delta--;
+ }
+ }
+ // Collect valid IDs
+ int collectedTags = 0;
+ if (origo < startTermPos) {
+ collectedTags = startTermPos - origo;
+ origo = startTermPos;
+ }
+ List tags = new ArrayList(
+ Math.max(0, Math.min(requestGroup.getMaxTags(), endTermPos-origo)));
+ for (int termPos = origo ;
+ termPos < endTermPos & collectedTags < requestGroup.getMaxTags() ;
+ termPos++) {
+ if (tagCounts[termPos] >= minCount) {
+ tags.add(new FacetResponse.Tag(
+ map.getOrderedTerm(termPos).utf8ToString(), tagCounts[termPos]));
+ collectedTags++;
+ }
+ }
+ FacetResponse.Group responseGroup =
+ new FacetResponse.Group(requestGroup, tags);
+ extractionTime = System.currentTimeMillis() - extractionTime;
+ responseGroup.setExtractionTime(extractionTime);
+ responseGroup.setPotentialTags(endTermPos - startTermPos);
+
+ return responseGroup;
+ }
+
+ private FacetResponse.Group extractCountResult(
+ FacetRequest.FacetGroup requestGroup, int startTermPos, int endTermPos)
+ throws IOException {
+ long extractionTime = System.currentTimeMillis();
+ // Sort tag references by count
+ final int maxSize = Math.min(
+ endTermPos- startTermPos, requestGroup.getMaxTags());
+ final int minCount = requestGroup.getMinCount();
+ ExposedPriorityQueue pq = new ExposedPriorityQueue(
+ new ExposedComparators.IndirectComparator(tagCounts), maxSize);
+ long totalRefs = 0;
+ long totalValidTags = 0;
+ for (int termPos = startTermPos ; termPos < endTermPos ; termPos++) {
+ totalRefs += tagCounts[termPos];
+ if (tagCounts[termPos] >= minCount) {
+ pq.insertWithOverflow(termPos);
+ totalValidTags++;
+ }
+ }
+
+ // Extract Tags
+ FacetResponse.Tag[] tags = new FacetResponse.Tag[pq.size()];
+ int pos = pq.size()-1;
+ while (pq.size() > 0) {
+ final int termIndex = pq.pop();
+ tags[pos--] = new FacetResponse.Tag(
+ map.getOrderedTerm(termIndex).utf8ToString(), tagCounts[termIndex]);
+ }
+
+ // Create response
+ FacetResponse.Group responseGroup =
+ new FacetResponse.Group(requestGroup, Arrays.asList(tags));
+ extractionTime = System.currentTimeMillis() - extractionTime;
+ responseGroup.setExtractionTime(extractionTime);
+ responseGroup.setPotentialTags(endTermPos-startTermPos);
+ responseGroup.setTotalReferences(totalRefs);
+ responseGroup.setValidTags(totalValidTags);
+ return responseGroup;
+ }
+
+ /**
+ * The query is typically used for cache purposes.
+ * @param query the query used for the collector filling search.
+ */
+ public void setQuery(String query) {
+ this.query = query;
+ }
+
+ /**
+ * @return the query used for filling this collector. Must be set explicitly
+ * by the user of this class with {@link #setQuery(String)}.
+ */
+ public String getQuery() {
+ return query;
+ }
+
+ /**
+ * This is set implicitely by calls to {@link #collect(int[])} and
+ * {@link #collect(OpenBitSet)} but must be set explicitly with
+ * {@link #setCountTime(long)} if the TagCollector is used as a Lucene
+ * collector with a standard search.
+ * @return the number of milliseconds used for filling the tag counter.
+ */
+ public long getCountTime() {
+ return countTime;
+ }
+
+ /**
+ * @param countTime the number of milliseconds used for filling the counter.
+ */
+ public void setCountTime(long countTime) {
+ this.countTime = countTime;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java Sat Sep 18 01:19:34 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java Sat Sep 18 01:19:34 CEST 2010
@@ -0,0 +1,150 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Order-oriented variation on {@link org.apache.lucene.index.Terms}.
+ *
+ * Access is primarily through ordinals. This can be done directly with
+ * {@link #getTerm} and {@link #getField} or indirectly through
+ * {@link #getOrderedTerm} and {@link #getOrderedField}. Using the ordered
+ * methods is equivalent to {@code getTerm(getOrder().get(orderPosition))}.
+ */
+public interface TermProvider {
+
+ /**
+ * Performs a search for the given term, returning the indirect for the
+ * term or the nearest indirect if not present.
+ *
+ * Note: While a binary search is used, this is still somewhat expensive as
+ * the terms themselves needs to be retrieved.
+ * @param key the term to search for.
+ * @return the indirect for the best matching term.
+ * @throws java.io.IOException if the terms could not be accessed.
+ */
+ public int getNearestTermIndirect(BytesRef key) throws IOException;
+
+ /**
+ * Performs a search for the given term, returning the indirect for the
+ * term or the nearest indirect if not present.
+ *
+ * Note: While a binary search is used, this is still somewhat expensive as
+ * the terms themselves needs to be retrieved.
+ * @param key the term to search for.
+ * @param startTermPos where to search from (inclusive).
+ * @param endTermPos where to search to (exclusive).
+ * @return the indirect for the best matching term.
+ * @throws java.io.IOException if the terms could not be accessed.
+ */
+ public int getNearestTermIndirect(
+ BytesRef key, int startTermPos, int endTermPos) throws IOException;
+
+ /**
+ * @return a comparator used for sorting. If this is null, natural BytesRef
+ * order is to be used.
+ */
+ Comparator getComparator();
+
+ /**
+ * @return an ID identifying the comparator. If natural BytesRef order is to
+ * be used, the ID should be {@link ExposedRequest#LUCENE_ORDER}.
+ */
+ String getComparatorID();
+
+ /**
+ * @return a short name or description of this provider.
+ */
+ String getDesignation(); // Debugging and feedback
+
+ String getField(long ordinal) throws IOException;
+
+ BytesRef getTerm(long ordinal) throws IOException;
+
+ String getOrderedField(long indirect) throws IOException;
+
+ // Note: indirect might be -1 which should return null
+ BytesRef getOrderedTerm(long indirect) throws IOException;
+
+ long getUniqueTermCount() throws IOException;
+
+ long getOrdinalTermCount() throws IOException;
+
+ long getMaxDoc();
+
+ IndexReader getReader() throws IOException;
+
+ int getReaderHash();
+
+ /**
+ * If the provider is a leaf, the hash-value is that of the IndexReader.
+ * If the provider is a node, the value is the sum of all contained
+ * term-providers hash-values.
+ *
+ * This is useful for mapping segment-level docIDs to index-level then a
+ * group consists of more than one segment.
+ * @return a hash-value useful for determining reader-equivalence.
+ */
+ int getRecursiveHash();
+
+ /**
+ * The ordinals sorted by a previously provided Comparator (typically a
+ * Collator).
+ * @return ordinals ordered by Comparator..
+ * @throws IOException if the ordinals could not be retrieved.
+ */
+ PackedInts.Reader getOrderedOrdinals() throws IOException;
+
+ /**
+ * Mapping from docID to the ordered ordinals returned by
+ * {@link #getOrderedOrdinals()}. Usable for sorting by field value as
+ * comparisons of documents can be done with {@code
+ PackedInts.Reader doc2indirect = getDocToSingleIndirect();
+ ...
+ if (doc2indirect.get(docID1) < doc2indirect.get(docID2)) {
+ System.out.println("Doc #" + docID1 + " comes before doc # " + docID2);
+ }}. If the term for a document is needed it can be retrieved with
+ * {@code getOrderedTerm(doc2indirect.get(docID))}.
+ *
+ * Documents without a corresponding term has the indirect value {@code -1}.
+ * @return a map from document IDs to the indirect values for the terms for
+ * the documents.
+ * @throws IOException if the index could not be accessed,
+ */
+ // TODO: Handle missing value (-1? Max+1? Defined by "empty first"?)
+ PackedInts.Reader getDocToSingleIndirect() throws IOException;
+
+ /**
+ * @param collectDocIDs if true, the document IDs are returned as part of the
+ * tuples. Note that this normally increases the processing time
+ * significantly.
+ * @return an iterator over tuples in the order defined for the TermProvider.
+ * @throws IOException if the iterator could not be constructed.
+ */
+ Iterator getIterator(boolean collectDocIDs) throws IOException;
+
+ DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException;
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java Fri Sep 17 21:05:50 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java Fri Sep 17 21:05:50 CEST 2010
@@ -0,0 +1,71 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A representation of Field, Term, ordinal, indirect and potentially docID.
+ */
+public class ExposedTuple {
+ public String field;
+ /**
+ * The term itself. Note that this is re-used by the iterator, so users of
+ * the iterator must finish processing the current term before calling next.
+ */
+ public BytesRef term;
+ /**
+ * The ordinal for the term, used for direct lookup.
+ */
+ public long ordinal;
+ /**
+ * The indirect for the term, used for lookup with
+ * {@link TermProvider#getOrderedTerm(long)}.
+ */
+ public long indirect;
+ /**
+ * A docID for the term or -1 if docIDs are not requested or not existing.
+ * If docIDs are requested and if there are multiple docIDs for the term,
+ * multiple ExposedTuples with the same term and the same ordinal will be
+ * used.
+ */
+ public long docID;
+
+ /*
+ * The order of the term, relative to the other Terms delivered, when using
+ * an Iterator from {@link #getExposedTuples}. Starts at 0, increments by at
+ * most 1 for each subsequent ExposedTuple from the iterator.
+ */
+ //public long order;
+
+ public ExposedTuple(
+ String field, BytesRef term, long ordinal, long indirect, long docID) {
+ this.field = field;
+ this.term = term;
+ this.ordinal = ordinal;
+ this.indirect = indirect;
+ this.docID = docID;
+ }
+
+ public ExposedTuple(ExposedTuple other) {
+ this.field = other.field;
+ this.term = other.term;
+ this.ordinal = other.ordinal;
+ this.indirect = other.indirect;
+ this.docID = other.docID;
+ }
+
+ public String toString() {
+ return "ExposedTuple(" + field + ":"
+ + (term == null ? "null" : term.utf8ToString()) + ", ord=" + ordinal
+ + ", indirect=" + indirect + ", docID=" + docID + ")";
+ }
+
+ // Convenience
+ public void set(
+ String field, BytesRef term, long ordinal, long indirect, long docID) {
+ this.field = field;
+ this.term = term;
+ this.ordinal = ordinal;
+ this.indirect = indirect;
+ this.docID = docID;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml Sat Sep 18 18:07:44 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/facet/FacetResponse.xml Sat Sep 18 18:07:44 CEST 2010
@@ -0,0 +1,27 @@
+
+
+
+
+ 00000006
+ 00000002
+ 00000000
+ 00000004
+ 09999998
+
+
+ a_ fOO01991201
+ a_FOO0qVi Pljvjbæ9v5578717
+ a_FoO1725831
+ a_ foo18a01AøoCf 4518220
+ a_FOO1hzW5450992
+
+
+
+
+ D
+ E
+ F
+ G
+ H
+
+
Index: lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java Thu Sep 23 13:48:48 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java Thu Sep 23 13:48:48 CEST 2010
@@ -0,0 +1,456 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.IdentityReader;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.Iterator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Keeps track of a single field in a single segment. Optimized towards
+ * SegmentReaders although it can be used with other IndexReaders as long as
+ * they are leafs (has no sub readers).
+ *
+ * The provider is lazy meaning that the ordering is only calculated when
+ * needed.
+ */
+public class FieldTermProvider extends TermProviderImpl {
+ private final IndexReader reader; // This is treated as a SegmentReader
+ private ExposedRequest.Field request;
+ private PackedInts.Reader order = null;
+
+
+ // TODO: Consider letting the cache be a factor of term count
+ /**
+ * The minimum number of Terms or ExtendedTerms to hold in cache during
+ * processing.
+ * The allocated amount of memory depends on average term size and whether
+ * CollatorKey-based sorting is used. With an average term-length of 20
+ * characters, each cache-entry takes approximately 100 bytes without
+ * CollatorKey and 300 with CollatorKey.
+ *
+ * .
+ * 10.000 is thus roughly equivalent to either 1 or 3 MB of RAM depending on
+ * Collator presence.
+ */
+ static int minSortCacheSize = 20000; // 2-6 MB
+ static double sortCacheFraction = 0.05; // 1M docs = 5-15 MB
+ static int maxSortCacheSize = 300000; // 30-90 MB
+ private int sortCacheSize;
+
+ static int mergeChunkFragmentationFactor = 3; // TODO: Experiment with this
+ static int iteratorCacheSize = 5000; // TODO: Make this relative
+ static int iteratorReadAhead = 100;
+
+ /**
+ * If this is true and the given comparator is a
+ * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized
+ * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the
+ * chunk-sort speed.
+ */
+ public static boolean optimizeCollator = true;
+
+ private Terms terms;
+ private TermsEnum termsEnum;
+
+ public FieldTermProvider(IndexReader reader, ExposedRequest.Field request,
+ boolean cacheTables) throws IOException {
+ super(request.getComparator(), request.getComparatorID(),
+ "Field " + request.getField(), cacheTables);
+ if (reader.getSequentialSubReaders() != null) {
+ throw new IllegalArgumentException(
+ "The IndexReader should be a leaf (no sub readers). It contained "
+ + reader.getSequentialSubReaders().length + " sub readers");
+ }
+ this.reader = reader;
+ this.request = request;
+
+ terms = reader.fields().terms(request.getField());
+ termsEnum = terms == null ? null : terms.iterator(); // It's okay to be empty
+ sortCacheSize = getSortCacheSize(reader);
+ }
+
+ public ExposedRequest.Field getRequest() {
+ return request;
+ }
+
+ public String getField(long ordinal) {
+ return request.getField();
+ }
+
+ private static final int ITERATE_LIMIT = 10;
+ private long lastOrdinalRequest = -1;
+ public synchronized BytesRef getTerm(final long ordinal) throws IOException {
+ if (termsEnum == null) {
+ throw new IOException("No terms for field " + request.getField()
+ + " in segment " + reader + ". Requested ordinal was " + ordinal);
+ }
+ // TODO: Upstream this simple sequential access optimization
+ if (lastOrdinalRequest <= ordinal &&
+ ordinal <= lastOrdinalRequest + ITERATE_LIMIT) {
+ BytesRef term = termsEnum.term();
+ while (lastOrdinalRequest != ordinal) {
+ term = termsEnum.next();
+ if (term == null) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+ lastOrdinalRequest++;
+ }
+ lastOrdinalRequest = ordinal;
+ return new BytesRef(term);
+ }
+
+ if (TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+ BytesRef result = termsEnum.term();
+ lastOrdinalRequest = ordinal;
+ return new BytesRef(result);
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ if (termsEnum == null) {
+ throw new IOException("No terms or docIDs for field " + request.getField()
+ + " in segment " + reader + ". Requested ordinal was " + ordinal);
+ }
+ if (ordinal != lastOrdinalRequest &&
+ TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+
+ lastOrdinalRequest = ordinal;
+ //
+ return termsEnum.docs(reader.getDeletedDocs(), reuse);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return request.getField();
+ }
+
+ public BytesRef getOrderedTerm(final long indirect) throws IOException {
+ return indirect == -1 ? null :
+ getTerm(getOrderedOrdinals().get((int)indirect));
+ }
+
+ public long getUniqueTermCount() throws IOException {
+ return terms == null ? 0 : terms.getUniqueTermCount();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return terms == null ? 0 : terms.getUniqueTermCount();
+ }
+
+ public long getMaxDoc() {
+ return reader.maxDoc();
+ }
+
+ public IndexReader getReader() throws IOException {
+ return reader;
+ }
+
+ public int getReaderHash() {
+ return reader.hashCode();
+ }
+
+ public int getRecursiveHash() {
+ return reader.hashCode();
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ if (termsEnum == null) {
+ return ExposedSettings.getMutable(0, 1);
+ }
+
+ if (order != null) {
+ return order;
+ }
+ PackedInts.Reader newOrder;
+ if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID())) {
+ newOrder = new IdentityReader((int)terms.getUniqueTermCount());
+ } else {
+ newOrder = sortOrdinals();
+ }
+
+ if (cacheTables) {
+ order = newOrder;
+ }
+ return newOrder;
+ }
+
+ public Iterator getIterator(
+ boolean collectDocIDs) throws IOException {
+ PackedInts.Reader order = getOrderedOrdinals();
+ if (collectDocIDs) {
+ return new TermDocIterator(this, true);
+ }
+ this.order = order;
+ CachedTermProvider cache = new CachedTermProvider(
+ this, iteratorCacheSize, iteratorReadAhead);
+ return new TermDocIterator(cache, false);
+ }
+
+ private PackedInts.Reader sortOrdinals() throws IOException {
+ int termCount = (int)getOrdinalTermCount();
+ int[] ordered = new int[termCount];
+ for (int i = 0 ; i < termCount ; i++) {
+ ordered[i] = i; // Initial order = Lucene
+ }
+
+ long startTime = System.nanoTime();
+ sort(ordered);
+ long sortTime = (System.nanoTime() - startTime);
+
+
+ // TODO: Remove this
+ System.out.println("Chunk total sort for field " + getField(0)
+ + ": " + ExposedUtil.time("terms", ordered.length, sortTime / 1000000));
+/* System.out.println(String.format(
+ "Sorted %d Terms in %s out of which %s (%s%%) was lookups and " +
+ "%s (%s%%) was collation key creation. " +
+ "The cache (%d terms) got %d requests with %d (%s%%) misses",
+ termCount, nsToString(sortTime),
+ nsToString(lookupTime),
+ lookupTime * 100 / sortTime,
+ nsToString(collatorKeyCreation),
+ collatorKeyCreation * 100 / sortTime,
+ sortCacheSize, cacheRequests,
+ cacheMisses, cacheMisses * 100 / cacheRequests));
+ */
+ PackedInts.Mutable packed =
+ ExposedSettings.getMutable(termCount, termCount);
+ for (int i = 0 ; i < termCount ; i++) {
+ packed.set(i, ordered[i]); // Save space by offsetting min to 0
+ }
+ return packed;
+ }
+
+ /*
+ * Starts by dividing the ordered array in logical chunks, then sorts each
+ * chunk separately and finishes by merging the chunks.
+ */
+ private void sort(final int[] ordinals) throws IOException {
+ int chunkSize = Math.max(sortCacheSize, ordinals.length / sortCacheSize);
+ int chunkCount = (int) Math.ceil(ordinals.length * 1.0 / chunkSize);
+
+ // We do not thread the sort as the caching of Strings is more important
+ // than processing power.
+
+ // We sort in chunks so the cache is 100% effective
+ if (optimizeCollator && request.getComparator() instanceof
+ ExposedComparators.BytesRefWrappedCollator) {
+// System.out.println("Using CollatorKey optimized chunk sort");
+ optimizedChunkSort(ordinals, chunkSize);
+ } else {
+ chunkSort(ordinals, chunkSize);
+ }
+
+ if (chunkSize >= ordinals.length) {
+ return; // Only one chunk. No need for merging
+ }
+
+ // We have sorted chunks. Commence the merging!
+
+ chunkMerge(ordinals, chunkSize, chunkCount);
+ }
+
+ private void chunkMerge(
+ int[] ordinals, int chunkSize, int chunkCount) throws IOException {
+ // Merging up to cache-size chunks requires an efficient way to determine
+ // the chunk with the lowest value. As locality is not that important with
+ // all comparables in cache, we use a heap.
+ // The heap contains an index (int) for all active chunks in the term order
+ // array. When an index is popped, it is incremented and re-inserted unless
+ // it is a block start index in which case it is just discarded.
+
+ System.out.println("Beginning merge sort of " + ordinals.length
+ + " ordinals in " + chunkCount + " chunks");
+ long mergeTime = System.currentTimeMillis();
+ CachedProvider cache;
+ ExposedComparators.OrdinalComparator indirectComparator;
+ if (optimizeCollator && request.getComparator() instanceof
+ ExposedComparators.BytesRefWrappedCollator) {
+ Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request.
+ getComparator()).getCollator();
+ CachedCollatorKeyProvider keyCache = new CachedCollatorKeyProvider(
+ this, collator, sortCacheSize, chunkSize-1);
+ keyCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount /
+ mergeChunkFragmentationFactor));
+ keyCache.setOnlyReadAheadIfSpace(true);
+ keyCache.setStopReadAheadOnExistingValue(true);
+ indirectComparator = ExposedComparators.wrapIndirect(keyCache, ordinals);
+ cache = keyCache;
+ } else {
+ CachedTermProvider termCache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ // Configure the cache so that read ahead is lower as the access pattern
+ // to the ordinals are not guaranteed linear
+ termCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount /
+ mergeChunkFragmentationFactor));
+ termCache.setOnlyReadAheadIfSpace(true);
+ termCache.setStopReadAheadOnExistingValue(true);
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ termCache.getTerm(i); // Warm cache
+ }
+ indirectComparator = ExposedComparators.wrapIndirect(
+ termCache, ordinals, request.getComparator());
+ cache = termCache;
+ }
+
+ ExposedPriorityQueue pq = new ExposedPriorityQueue(
+ indirectComparator, chunkCount);
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ pq.add(i);
+ }
+
+ int[] sorted = new int[ordinals.length];
+ for (int i = 0 ; i < sorted.length ; i++) {
+ Integer next = pq.pop();
+ if (next == -1) {
+ throw new IllegalStateException(
+ "Popping the heap should never return -1");
+ }
+ sorted[i] = ordinals[next];
+ cache.release(sorted[i]); // Important for cache read ahead efficiency
+ if (++next % chunkSize != 0 && next != ordinals.length) {
+ pq.add(next);
+ }
+ }
+ mergeTime = System.currentTimeMillis() - mergeTime;
+ System.out.println(
+ "Merged " + ExposedUtil.time("chunks", chunkCount, mergeTime)
+ + " aka " + ExposedUtil.time("terms", sorted.length, mergeTime)
+ + ": " + cache.getStats());
+ System.arraycopy(sorted, 0, ordinals, 0, sorted.length);
+/* System.out.println(String.format(
+ "Heap merged %d sorted chunks of size %d (cache: %d, total terms: %s)" +
+ " in %s with %d cache misses (%d combined for both sort passes)",
+ chunkCount, chunkSize, sortCacheSize, ordered.length,
+ nsToString(System.nanoTime() - startTimeHeap),
+ cacheMisses - oldHeapCacheMisses, cacheMisses - oldCacheMisses));*/
+ }
+
+ private void chunkSort(int[] ordinals, int chunkSize) throws IOException {
+ CachedTermProvider cache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ // Sort the chunks individually
+ //long startTimeMerge = System.nanoTime();
+ ExposedComparators.OrdinalComparator comparator =
+ ExposedComparators.wrap(cache, request.getComparator());
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ long chunkTime = System.currentTimeMillis();
+ cache.getTerm(i); // Tim-sort starts at 1 so we init the read-ahead at 0
+ ExposedTimSort.sort(
+ ordinals, i, Math.min(i + chunkSize, ordinals.length), comparator);
+ chunkTime = System.currentTimeMillis() - chunkTime;
+ int percent = (Math.min(i + chunkSize, ordinals.length)-1) *100
+ / ordinals.length;
+ System.out.println("Chunk sorted " + percent + "% " + i + "-"
+ + (Math.min(i + chunkSize, ordinals.length)-1) + ": "
+ + ExposedUtil.time("ordinals", Math.min(i + chunkSize,
+ ordinals.length) - i, chunkTime) + ": "+ cache.getStats());
+ cache.clear();
+ }
+ }
+
+ // Ordinals _must_ be monotonously increasing
+ private void optimizedChunkSort(
+ int[] ordinals, int chunkSize) throws IOException {
+ Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request.
+ getComparator()).getCollator();
+ CachedTermProvider cache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ CollatorPair[] keys = new CollatorPair[chunkSize];
+ for (int start = 0 ; start < ordinals.length ; start += chunkSize) {
+ long chunkTime = System.currentTimeMillis();
+
+ int end = Math.min(start + chunkSize, ordinals.length); // Exclusive
+ // Fill
+ for (int index = start ; index < end ; index++) {
+ keys[index - start] = new CollatorPair(
+ ordinals[index], collator.getCollationKey(
+ cache.getTerm(ordinals[index]).utf8ToString()));
+ }
+ // Sort
+ ExposedTimSort.sort(
+ ordinals, start, end, new TimComparator(start, keys));
+/* // Store
+ for (int index = start ; index < end ; index++) {
+ ordinals[index] = (int) keys[index - start].ordinal;
+ }*/
+
+ chunkTime = System.currentTimeMillis() - chunkTime;
+ int percent = (int) ((long)end * 100 / ordinals.length);
+ System.out.println("Chunk sorted " + percent + "% "
+ + start + "-" + (end-1) + ": "
+ + ExposedUtil.time("ordinals", end - start, chunkTime)
+ + ": "+ cache.getStats());
+ cache.clear();
+ }
+ }
+ private static final class TimComparator
+ implements ExposedComparators.OrdinalComparator {
+ private int start; // Inclusive
+ private CollatorPair[] keys;
+
+ public TimComparator(int start, CollatorPair[] keys) {
+ this.start = start;
+ this.keys = keys;
+ }
+
+ public int compare(int value1, int value2) {
+ return keys[value1 - start].compareTo(keys[value2 - start]);
+ }
+ }
+
+
+ private static final class CollatorPair implements Comparable{
+ private long ordinal;
+ private CollationKey key;
+
+ private CollatorPair(long ordinal, CollationKey key) {
+ this.ordinal = ordinal;
+ this.key = key;
+ }
+ public int compareTo(CollatorPair o) {
+ return key.compareTo(o.key);
+ }
+ public long getOrdinal() {
+ return ordinal;
+ }
+ }
+
+ /**
+ * Calculate the sort cache size as a fraction of the total number of
+ * documents in the reader,
+ * @param reader the reader to cache.
+ * @return a cache size that fits in scale to the reader.
+ */
+ private int getSortCacheSize(IndexReader reader) {
+ final int fraction = (int) (reader.maxDoc() * sortCacheFraction);
+ return Math.min(maxSortCacheSize, Math.max(minSortCacheSize, fraction));
+ }
+}
\ No newline at end of file