fields,
+ int fieldContentLength, int minSegments)
+ throws IOException {
+ long startTime = System.nanoTime();
+ File location = INDEX_LOCATION;
+ Random random = new Random(87);
+
+ Directory dir = FSDirectory.open(INDEX_LOCATION);
+ IndexWriter writer = new IndexWriter(
+ dir, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT,
+ new MockAnalyzer()));
+
+ writer.getConfig().setRAMBufferSizeMB(16.0);
+ for (int docID = 0 ; docID < docCount ; docID++) {
+ Document doc = new Document();
+ for (String field: fields) {
+ doc.add(new Field(
+ field,
+ field + "_" + getRandomString(
+ random, CHARS, 1, fieldContentLength) + docID,
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ }
+ doc.add(new Field(ID, ID_FORMAT.format(docID),
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.add(new Field(EVEN, docID % 2 == 0 ? "true" : "false",
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ if (docID % 2 == 1) {
+ doc.add(new Field(EVEN_NULL, getRandomString(
+ random, CHARS, 1, fieldContentLength) + docID,
+ Field.Store.YES, Field.Index.NOT_ANALYZED));
+ }
+ doc.add(new Field(ALL, ALL, Field.Store.YES, Field.Index.NOT_ANALYZED));
+ writer.addDocument(doc);
+ if (docID == docCount / minSegments) {
+ writer.commit(); // Ensure minSegments
+ }
+ }
+// writer.optimize();
+ writer.close();
+ System.out.println(String.format(
+ "Created %d document index with %d fields with average " +
+ "term length %d and total size %s in %sms at %s",
+ docCount, fields.size() + 2, fieldContentLength / 2,
+ readableSize(calculateSize(location)),
+ (System.nanoTime() - startTime) / 1000000, location.getAbsolutePath()));
+ }
+
+ private static StringBuffer buffer = new StringBuffer(100);
+ static synchronized String getRandomString(
+ Random random, char[] chars, int minLength, int maxLength) {
+ int length = minLength == maxLength ? minLength :
+ random.nextInt(maxLength-minLength+1) + minLength;
+ buffer.setLength(0);
+ for (int i = 0 ; i < length ; i++) {
+ buffer.append(chars[random.nextInt(chars.length)]);
+ }
+ return buffer.toString();
+ }
+
+ private String readableSize(long size) {
+ return size > 2 * 1048576 ?
+ size / 1048576 + "MB" :
+ size > 2 * 1024 ?
+ size / 1024 + "KB" :
+ size + "bytes";
+ }
+
+ private long calculateSize(File file) {
+ long size = 0;
+ if (file.isDirectory()) {
+ for (File sub: file.listFiles()) {
+ size += calculateSize(sub);
+ }
+ } else {
+ size += file.length();
+ }
+ return size;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/util/packed/Packed64.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/Packed64.java (revision 931327)
+++ lucene/src/java/org/apache/lucene/util/packed/Packed64.java Tue Aug 31 11:17:43 CEST 2010
@@ -177,7 +177,7 @@
* @return the value at the given index.
*/
public long get(final int index) {
- final long majorBitPos = index * bitsPerValue;
+ final long majorBitPos = (long)index * bitsPerValue;
final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
@@ -188,7 +188,7 @@
}
public void set(final int index, final long value) {
- final long majorBitPos = index * bitsPerValue;
+ final long majorBitPos = (long)index * bitsPerValue;
final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
final int base = bitPos * FAC_BITPOS;
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/README.TXT
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/README.TXT Thu Sep 02 14:19:41 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/README.TXT Thu Sep 02 14:19:41 CEST 2010
@@ -0,0 +1,11 @@
+The while poc-folder is just for testing. The Mock-classes are copied from the
+test-folder in order to be accessible in the Lucene JAR.
+
+
+Get command line usage by calling
+java -cp build/lucene-core-4.0-dev.jar org.apache.lucene.search.exposed.poc.ExposedPOC
+
+Sample command line
+
+
+Note: This patch only supports Lucene flex indexes
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java Fri Aug 13 09:53:17 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedUtil.java Fri Aug 13 09:53:17 CEST 2010
@@ -0,0 +1,15 @@
+package org.apache.lucene.search.exposed;
+
+public class ExposedUtil {
+ private ExposedUtil() {
+ // Disables all constructors
+ }
+
+ public static String time(String type, long num, long ms) {
+ if (ms == 0) {
+ return num + " " + type + " in 0 ms";
+ }
+ return num + " " + type + " in " + ms + " ms: ~= "
+ + num/ms + " " + type + "/ms";
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java Mon Aug 09 09:53:38 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedRequest.java Mon Aug 09 09:53:38 CEST 2010
@@ -0,0 +1,212 @@
+/* $Id:$
+ *
+ * The Summa project.
+ * Copyright (C) 2005-2010 The State and University Library
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A request is three-tiered. At the upper level if encompasses multiple groups
+ * and normally mapping from documents to terms. The standard use-case is for
+ * faceting.
+ *
+ * Second tier is Group which is the primary access point for sorting. A Group
+ * contains one or more Fields from one or more segments.
+ *
+ * Third tier is Field which is segment-oriented. It is used internally by
+ * Group but can be used externally if the index is fully optimized and sorting
+ * is requested for a single field.
+ */
+public class ExposedRequest {
+ /**
+ * If the comparator key is set to the value "LUCENE", the order of the
+ * terms is Lucene's default order.
+ */
+ public static final String LUCENE_ORDER = "LUCENE";
+
+ private List groups;
+
+ public ExposedRequest(List groups) {
+ this.groups = groups;
+ }
+
+ public List getGroups() {
+ return groups;
+ }
+
+ /**
+ * Checks whether the given request is a true subset of this. If a request is
+ * a true subset, it can be processed by executing the superset and
+ * extracting only the parts specified in the subset.
+ *
+ * This is a trade-off as it takes longer to execute the superset.
+ * @param other the possible subset.
+ * @return true if other is a true subset.
+ */
+ public boolean encapsulates(ExposedRequest other) {
+ groupLoop:
+ for (Group otherGroup: other.getGroups()) {
+ for (Group thisGroup: getGroups()) {
+ if (thisGroup.equals(otherGroup)) {
+ continue groupLoop;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * A group is a collections of fields that are to be treated as one. An example
+ * could be a group named "title" with the fields "lti", "sort_title" and
+ * "subtitle" as fields. The same comparator must be used for all fields in
+ * order for term ordering to be consistent for the group.
+ */
+ public static class Group {
+ private String name;
+ private List fields;
+ private Comparator comparator;
+ private String comparatorID;
+
+ // Facet and sort-specific. Should it be here?
+ private int maxTerms;
+ private boolean nullFirst;
+
+ public Group(String name, List fields,
+ Comparator comparator, String comparatorID,
+ int maxTerms, boolean nullFirst) {
+ this.name = name;
+ this.fields = fields;
+ this.comparator = comparator;
+ this.comparatorID = comparatorID;
+
+ this.maxTerms = maxTerms;
+ this.nullFirst = nullFirst;
+ }
+
+ /**
+ * Checks whether the given Group is equal to . The order of the contained
+ * fields is irrelevant, but the set of fields must be equal.
+ *
+ * The name of the group is not used while comparing.
+ * @param other the group to compare to.
+ * @return true if the two groups are equivalent..
+ */
+ // TODO: Fix hashcode to fulfill the equals vs. hashcode contract
+ public boolean equals(Group other) {
+ if (other.getFields().size() != getFields().size() ||
+ !getComparatorID().equals(other.getComparatorID())) {
+ return false;
+ }
+
+ fieldLoop:
+ for (Field otherField: other.getFields()) {
+ for (Field thisField: getFields()) {
+ if (thisField.equals(otherField)) {
+ continue fieldLoop;
+ }
+ }
+ return false;
+ }
+ return true;
+ }
+
+
+ public String getName() {
+ return name;
+ }
+ public List getFields() {
+ return fields;
+ }
+ public Comparator getComparator() {
+ return comparator;
+ }
+ public String getComparatorID() {
+ return comparatorID;
+ }
+
+ public int getMaxTerms() {
+ return maxTerms;
+ }
+ }
+
+ public static class Field {
+ private String field;
+ private Comparator comparator;
+ private String comparatorID;
+
+ public Field(
+ String field, Comparator comparator, String comparatorID) {
+ this.field = field;
+ this.comparator = comparator;
+ this.comparatorID = comparatorID;
+ }
+
+ /**
+ * Checks whether the given object is also a Field and if so whether the
+ * field and comparatorIDs are equal.
+ * @param obj candidate for equality.
+ * @return true if the given object is equivalent to this.
+ */
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Field)) {
+ return false;
+ }
+ Field other = (Field)obj;
+ return field.equals(other.field)
+ && comparatorID.equals(other.comparatorID);
+ }
+
+ public String getField() {
+ return field;
+ }
+ public Comparator getComparator() {
+ return comparator;
+ }
+ public String getComparatorID() {
+ return comparatorID;
+ }
+
+ public boolean equals(Field otherField) {
+ return getField().equals(otherField.getField()) &&
+ getComparatorID().equals(otherField.getComparatorID());
+ }
+ }
+
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java Thu Aug 12 10:06:46 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestGroupTermProvider.java Thu Aug 12 10:06:46 CEST 2010
@@ -0,0 +1,180 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+public class TestGroupTermProvider extends LuceneTestCase {
+ public static final int DOCCOUNT = 10;
+ private ExposedHelper helper;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ helper.close();
+ }
+
+ public void testIndexGeneration() throws Exception {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ long termCount = 0;
+ TermsEnum terms = MultiFields.getFields(reader).
+ terms(ExposedHelper.ID).iterator();
+ while (terms.next() != null) {
+ assertEquals("The ID-term #" + termCount + " should be correct",
+ ExposedHelper.ID_FORMAT.format(termCount),
+ terms.term().utf8ToString());
+ termCount++;
+ }
+
+ assertEquals("There should be the right number of terms",
+ DOCCOUNT, termCount);
+ reader.close();
+ }
+
+ public void testOrdinalAccess() throws IOException {
+ String[] fieldNames = new String[]{"a"};
+ helper.createIndex( DOCCOUNT, Arrays.asList(fieldNames), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ for (String fieldName: fieldNames) {
+ // FIXME: The ordinals are not comparable to the exposed ones!
+ TermsEnum terms =
+ MultiFields.getFields(reader).terms(fieldName).iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ }
+ }
+
+ TermProvider groupProvider = ExposedFactory.createProvider(
+ reader, "TestGroup", Arrays.asList(fieldNames), null, "null");
+
+ ArrayList exposedOrdinals = new ArrayList(DOCCOUNT);
+ for (int i = 0 ; i < groupProvider.getOrdinalTermCount() ; i++) {
+ exposedOrdinals.add(groupProvider.getTerm(i).utf8ToString());
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedOrdinals.size());
+ // We sort as the order of the terms is not significant here
+ Collections.sort(plainExtraction);
+ Collections.sort(exposedOrdinals);
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedOrdinals.get(i));
+ }
+ reader.close();
+
+ }
+ public void testTermSortAllDefined() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ testTermSort(reader, Arrays.asList("a"));
+ reader.close();
+ }
+
+ public void testTermSortAllScarce() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ testTermSort(reader, Arrays.asList("even"));
+ reader.close();
+ }
+
+ private void testTermSort(IndexReader index, List fieldNames)
+ throws IOException {
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ // TODO: Make this handle multiple field names
+ TermsEnum terms =
+ MultiFields.getFields(index).terms(fieldNames.get(0)).iterator();
+ while (terms.next() != null) {
+ String next = terms.term().utf8ToString();
+ plainExtraction.add(next);
+// System.out.println("Default order term #"
+// + (plainExtraction.size() - 1) + ": " + next);
+ }
+ Collections.sort(plainExtraction, sorter);
+
+ TermProvider groupProvider = ExposedFactory.createProvider(
+ index, "a-group", fieldNames,
+ ExposedComparators.collatorToBytesRef(sorter), "da");
+
+ ArrayList exposedExtraction = new ArrayList(DOCCOUNT);
+ Iterator ei = groupProvider.getIterator(false);
+ int count = 0;
+ while (ei.hasNext()) {
+ String next = ei.next().term.utf8ToString();
+ exposedExtraction.add(next);
+// System.out.println("Exposed sorted term #" + count++ + ": " + next);
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedExtraction.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedExtraction.get(i));
+ }
+ }
+
+ public void testDocIDMapping() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plain =
+ new ArrayList(DOCCOUNT);
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ plain.add(new ExposedHelper.Pair(
+ docID, reader.document(docID).get("a"), sorter));
+// System.out.println("Plain access added " + plain.get(plain.size()-1));
+ }
+ Collections.sort(plain);
+
+ TermProvider index =
+ ExposedFactory.createProvider(reader, "docIDGroup", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(sorter), "foo");
+
+ ArrayList exposed =
+ new ArrayList(DOCCOUNT);
+ Iterator ei = index.getIterator(true);
+
+ while (ei.hasNext()) {
+ ExposedTuple next = ei.next();
+// System.out.println("Extracted exposed " + next);
+ exposed.add(new ExposedHelper.Pair(
+ next.docID, next.term.utf8ToString(), sorter));
+ }
+ Collections.sort(exposed);
+
+ assertEquals("The two docID->term maps should be of equal length",
+ plain.size(), exposed.size());
+ for (int i = 0 ; i < plain.size() ; i++) {
+// System.out.println("Sorted docID, term #" + i + ". Plain=" + plain.get(i)
+// + ", Exposed=" + exposed.get(i));
+ assertEquals("Mapping #" + i + " should be equal",
+ plain.get(i), exposed.get(i));
+ }
+ reader.close();
+ }
+
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java Fri Aug 13 15:27:29 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/GroupTermProvider.java Fri Aug 13 15:27:29 CEST 2010
@@ -0,0 +1,255 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.GrowingMutable;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Keeps track of a group containing 1 or more providers ensuring term
+ * uniqueness and ordering. The providers are normally either
+ * {@link FieldTermProvider} or {@link GroupTermProvider}.
+ *
+ * The standard use case calls for caching the GroupTermProvider at segment
+ * level as group setups rarely changes.
+ */
+// TODO: Extend to handle term ordinals > Integer.MAX_VALUE
+public class GroupTermProvider extends TermProviderImpl {
+ private List providers;
+ private ExposedRequest.Group request;
+ private final int readerHash;
+
+ private PackedInts.Reader order;
+
+ private long[] termOrdinalStarts; // Starting points for term ordinals in the
+ // providers list
+ // FIXME: this should be relative to segments, not providers.
+ //Current implementation is not valid for multiple fields in the same segment
+ private long[] docIDStarts;
+
+ public GroupTermProvider(int readerHash,
+ List providers, ExposedRequest.Group request,
+ boolean cacheTables) throws IOException {
+ super(cacheTables);
+ this.readerHash = readerHash;
+ this.providers = providers;
+ this.request = request;
+
+ calculateStarts();
+ }
+
+ /**
+ * The starts are straight forward for term ordinals as they are just mapped
+ * as one long list of ordinals in the source order. We can do there is no
+ * authoritative index-level ordinal mapping.
+ *
+ * Document IDs are a bit trickier as they are used at the index-level.
+ * We need to handle the case where more than one field from the same segment
+ * make up a group. In that case the docIDs should have the same start.
+ *
+ * Important: In order for the calculator to work, the sources must be
+ * monotonic increasing
+ * @throws IOException if the underlying providers failed.
+ */
+ private void calculateStarts() throws IOException {
+ long sanityCheck = 0;
+ termOrdinalStarts = new long[providers.size() + 1];
+ docIDStarts = new long[providers.size() + 1];
+
+ for (int i = 1 ; i <= providers.size() ; i++) {
+ termOrdinalStarts[i] =
+ termOrdinalStarts[i-1] + providers.get(i-1).getOrdinalTermCount();
+ sanityCheck += providers.get(i-1).getOrdinalTermCount();
+ }
+ if (sanityCheck > Integer.MAX_VALUE-1) {
+ throw new InternalError("There are " + sanityCheck + " terms in total in "
+ + "the underlying TermProviders which is more than the current limit "
+ + "of Integer.MAX_VALUE");
+ }
+
+ long lastHash = -1;
+ long docStart = 0;
+ for (int i = 0 ; i < providers.size() ; i++) {
+ docIDStarts[i] = docStart;
+ TermProvider provider = providers.get(i);
+ docStart += provider.getRecursiveHash() == lastHash ? 0 :
+ provider.getMaxDoc();
+ lastHash = provider.getRecursiveHash();
+ }
+ docIDStarts[docIDStarts.length-1] = docStart;
+ }
+
+ public ExposedRequest.Group getRequest() {
+ return request;
+ }
+
+ public String getField(long ordinal) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getField(adjustOrdinal(ordinal, providerIndex));
+ }
+
+ private int getProviderIndex(long ordinal) throws IOException {
+ for (int i = 1 ; i < termOrdinalStarts.length ; i++) {
+ if (ordinal < termOrdinalStarts[i]) {
+ return i-1;
+ }
+ }
+ throw new IllegalArgumentException("The term ordinal " + ordinal
+ + " is above the maximum " + (getOrdinalTermCount() - 1));
+ }
+
+ private long adjustOrdinal(long ordinal, int providerIndex) {
+ return providerIndex == 0 ? ordinal :
+ ordinal - termOrdinalStarts[providerIndex];
+ }
+
+ public synchronized BytesRef getTerm(long ordinal) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getTerm(adjustOrdinal(ordinal, providerIndex));
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ int providerIndex = getProviderIndex(ordinal);
+ return providers.get(providerIndex).
+ getDocsEnum(adjustOrdinal(ordinal, providerIndex), reuse);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return getField(getOrderedOrdinals().get((int)indirect));
+ }
+
+ public BytesRef getOrderedTerm(final long indirect) throws IOException {
+ return indirect == -1 ? null :
+ getTerm(getOrderedOrdinals().get((int)indirect));
+ }
+
+ /**
+ * Note that this method calculates ordered ordinals to determine the unique
+ * terms.
+ * @return The number of unique terms.
+ * @throws IOException if the underlyind providers failed.
+ */
+ public long getUniqueTermCount() throws IOException {
+ return getOrderedOrdinals().size();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return termOrdinalStarts[termOrdinalStarts.length-1];
+ }
+
+ public long getMaxDoc() {
+ return docIDStarts[docIDStarts.length-1];
+ }
+
+ public IndexReader getReader() throws IOException {
+ throw new UnsupportedOperationException(
+ "Cannot request a reader from a collection of readers");
+ }
+
+ public int getReaderHash() {
+ return readerHash;
+ }
+
+ public int getRecursiveHash() {
+ int hash = 0;
+ for (TermProvider provider: providers) {
+ hash += provider.hashCode();
+ }
+ return hash;
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ if (order != null) {
+ return order;
+ }
+ PackedInts.Reader newOrder;
+/* if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID())) {
+ // TODO: This produces duplicates! We need to sort
+ newOrder = new IdentityReader((int)getOrdinalTermCount());
+ } else {*/
+ newOrder = sortOrdinals();
+ //}
+
+ if (cacheTables) {
+ order = newOrder;
+ }
+ return newOrder;
+ }
+
+ private PackedInts.Reader sortOrdinals() throws IOException {
+// System.out.println("Group sorting ordinals from " + providers.size()
+// + " providers");
+ int maxTermCount = (int)termOrdinalStarts[termOrdinalStarts.length-1];
+ long iteratorConstruction = System.currentTimeMillis();
+ GrowingMutable collector = new GrowingMutable(
+ 0, maxTermCount, 0, maxTermCount); // TODO: Why -1?
+ Iterator iterator = getIterator(false);
+ iteratorConstruction = System.currentTimeMillis() - iteratorConstruction;
+ System.out.println("Group total iterator construction: "
+ + ExposedUtil.time("ordinals", maxTermCount, iteratorConstruction));
+
+// int uniqueTermCount = 0;
+ long extractionTime = System.currentTimeMillis();
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+// System.out.println("sortOrdinals " + tuple + " term = " + tuple.term.utf8ToString() + " lookup term " + getTerm(tuple.ordinal).utf8ToString());
+ collector.set((int)tuple.indirect, tuple.ordinal);
+// collector.set(uniqueTermCount++, tuple.indirect);
+ }
+// System.out.println("Sorted merged term ordinals to " + collector);
+ // TODO: Check why the optimization below fails
+/* PackedInts.Mutable result = PackedInts.getMutable(
+ uniqueTermCount, PackedInts.bitsRequired(maxTermCount));
+ for (int i = 0 ; i < uniqueTermCount ; i++) {
+ result.set(i, collector.get(i));
+ }
+ return result;*/
+ extractionTime = System.currentTimeMillis() - extractionTime;
+ System.out.println("Group ordinal iterator depletion from "
+ + providers.size() + " providers: "
+ + ExposedUtil.time("ordinals", collector.size(), extractionTime));
+ return collector;
+ }
+
+
+ public Iterator getIterator(boolean collectDocIDs)
+ throws IOException {
+ return new MergingTermDocIterator(
+ this, providers, request.getComparator(), collectDocIDs);
+ }
+
+ public long segmentToIndexDocID(int providerIndex, long segmentDocID) {
+ return docIDStarts[providerIndex] + segmentDocID;
+ }
+
+ public long segmentToIndexTermOrdinal(
+ int providerIndex, long segmentTermOrdinal) {
+ return termOrdinalStarts[providerIndex] + segmentTermOrdinal;
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/CachedProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/CachedProvider.java Fri Aug 13 13:22:11 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/CachedProvider.java Fri Aug 13 13:22:11 CEST 2010
@@ -0,0 +1,156 @@
+package org.apache.lucene.search.exposed;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public abstract class CachedProvider {
+ private Map cache;
+ private int cacheSizeNum;
+ private int readAheadNum;
+ private long finalIndex; // Do not read past this
+
+ private boolean onlyReadAheadIfSpace = false;
+ private boolean stopReadAheadOnExistingValue = true;
+
+ // Stats
+ private long requests = 0;
+ private long lookups = 0;
+ private long readAheadRequests = 0;
+ private long misses = 0;
+ private long lookupTime = 0;
+
+ protected CachedProvider(int size, int readAhead, long finalIndex) {
+ this.cacheSizeNum = size;
+ this.readAheadNum = readAhead;
+ this.finalIndex = finalIndex;
+ cache = new LinkedHashMap(size, 1.2f, false) {
+ @Override
+ protected boolean removeEldestEntry(
+ Map.Entry eldest) {
+ return size() > cacheSizeNum;
+ }
+ };
+ }
+
+ protected abstract T lookup(final long index) throws IOException;
+
+ public T get(final long index) throws IOException {
+ requests++;
+ T entry = cache.get(index);
+ if (entry != null) {
+ return entry;
+ }
+ misses++;
+
+ long startLookup = System.nanoTime();
+ entry = lookup(index);
+ lookupTime += System.nanoTime() - startLookup;
+ lookups++;
+ cache.put(index, entry);
+ readAhead(index+1, readAheadNum);
+ return entry;
+
+ }
+
+ private void readAhead(final long startIndex, final int readAhead)
+ throws IOException {
+ long index = startIndex;
+ int readsLeft = readAhead;
+ while (true) {
+ if (readsLeft == 0 || index > finalIndex||
+ (onlyReadAheadIfSpace && cache.size() >= cacheSizeNum)) {
+ break;
+ }
+ readAheadRequests++;
+ T entry = cache.get(index);
+ if (entry == null) {
+ long startLookup = System.nanoTime();
+ entry = lookup(index);
+ lookupTime += System.nanoTime() - startLookup;
+ lookups++;
+ if (entry == null) {
+ break;
+ }
+/* if (index > 19999) {
+ System.out.println("Putting " + index);
+ }*/
+ cache.put(index, entry);
+ } else if (stopReadAheadOnExistingValue) {
+ break;
+ }
+ index++;
+ readsLeft--;
+ }
+ }
+
+ /**
+ * Removes the cache entry for the given index from cache is present.
+ *
+ * If the user of the cache knows that the String for a given ordinal should
+ * not be used again, calling this method helps the cache to perform better.
+ * @param index the ordinal for the elementto remove.
+ * @return the old element if it was present in the cache.
+ */
+ public T release(final long index) {
+ return cache.remove(index);
+ }
+
+ /**
+ * Clears the cache.
+ */
+ public void clear() {
+ cache.clear();
+ requests = 0;
+ lookups = 0;
+ readAheadRequests = 0;
+ misses = 0;
+ lookupTime = 0;
+ }
+
+ public int getCacheSize() {
+ return cacheSizeNum;
+ }
+
+ /**
+ * Setting the cache size clears the cache.
+ * @param cacheSize the new size of the cache, measured in elements.
+ */
+ public void setCacheSize(int cacheSize) {
+ this.cacheSizeNum = cacheSize;
+ cache.clear();
+ }
+
+ public int getReadAhead() {
+ return readAheadNum;
+ }
+
+ public void setReadAhead(int readAhead) {
+ this.readAheadNum = readAhead;
+ }
+
+ public boolean isOnlyReadAheadIfSpace() {
+ return onlyReadAheadIfSpace;
+ }
+
+ public void setOnlyReadAheadIfSpace(boolean onlyReadAheadIfSpace) {
+ this.onlyReadAheadIfSpace = onlyReadAheadIfSpace;
+ }
+
+ public boolean isStopReadAheadOnExistingValue() {
+ return stopReadAheadOnExistingValue;
+ }
+
+ public void setStopReadAheadOnExistingValue(
+ boolean stopReadAheadOnExistingValue) {
+ this.stopReadAheadOnExistingValue = stopReadAheadOnExistingValue;
+ }
+
+ public String getStats() {
+ return "CachedProvider cacheSize=" + cacheSizeNum
+ + ", misses=" + misses + "/" + requests
+ + ", lookups=" + lookups + " (" + lookupTime / 1000000 + " ms ~= "
+ + (lookupTime == 0 ? "N/A" : lookups * 1000000 / lookupTime)
+ + " lookups/ms), readAheads=" + readAheadRequests;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java Fri Aug 13 14:58:20 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/CachedCollatorKeyProvider.java Fri Aug 13 14:58:20 CEST 2010
@@ -0,0 +1,31 @@
+package org.apache.lucene.search.exposed;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+
+public class CachedCollatorKeyProvider extends CachedProvider {
+ private TermProvider source;
+ private final Collator collator;
+
+ /**
+ * @param source the backing term provider.
+ * @param collator the Collator used for key generation.
+ * @param cacheSize the maximum number of elements to hold in cache.
+ * @param readAhead the maximum number of lookups that can be performed
+ * after a plain lookup.
+ * @throws java.io.IOException if the cache could access the source.
+ */
+ public CachedCollatorKeyProvider(
+ TermProvider source, Collator collator,
+ int cacheSize, int readAhead) throws IOException {
+ super(cacheSize, readAhead, source.getOrdinalTermCount()-1);
+ this.source = source;
+ this.collator = collator;
+ }
+
+ @Override
+ protected CollationKey lookup(final long index) throws IOException {
+ return collator.getCollationKey(source.getTerm(index).utf8ToString());
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java Thu Aug 12 13:57:51 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/MockAnalyzer.java Thu Aug 12 13:57:51 CEST 2010
@@ -0,0 +1,94 @@
+package org.apache.lucene.search.exposed.poc;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Analyzer for testing
+ */
+public final class MockAnalyzer extends Analyzer {
+ private final CharacterRunAutomaton runAutomaton;
+ private final boolean lowerCase;
+ private final CharacterRunAutomaton filter;
+ private final boolean enablePositionIncrements;
+
+ /**
+ * Creates a new MockAnalyzer.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ * @param filter DFA describing how terms should be filtered (set of stopwords, etc)
+ * @param enablePositionIncrements true if position increments should reflect filtered terms.
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ this.runAutomaton = runAutomaton;
+ this.lowerCase = lowerCase;
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ /**
+ * Creates a new MockAnalyzer, with no filtering.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false);
+ }
+
+ /**
+ * Create a Whitespace-lowercasing analyzer with no stopwords removal
+ */
+ public MockAnalyzer() {
+ this(MockTokenizer.WHITESPACE, true);
+ }
+
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ return new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
+ }
+
+ private class SavedStreams {
+ MockTokenizer tokenizer;
+ MockTokenFilter filter;
+ }
+
+ @Override
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
+ SavedStreams saved = (SavedStreams) getPreviousTokenStream();
+ if (saved == null) {
+ saved = new SavedStreams();
+ saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
+ setPreviousTokenStream(saved);
+ return saved.filter;
+ } else {
+ saved.tokenizer.reset(reader);
+ return saved.filter;
+ }
+ }
+}
\ No newline at end of file
Index: lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
===================================================================
--- lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java (revision 984968)
+++ lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java Thu Sep 02 14:00:50 CEST 2010
@@ -232,4 +232,24 @@
mutable.set(4, 16);
assertEquals("The value #24 should remain unchanged", 31, mutable.get(24));
}
+
+ /*
+ Check if the structures properly handle the case where
+ index * bitsPerValue > Integer.MAX_VALUE
+ */
+ public void testIntOverflow() {
+ int INDEX = (int)Math.pow(2, 30)+1;
+ int BITS = 2;
+
+ Packed32 p32 = new Packed32(INDEX, BITS);
+ p32.set(INDEX-1, 1);
+ assertEquals("The value at position " + (INDEX-1)
+ + " should be correct for Packed32", 1, p32.get(INDEX-1));
+ p32 = null; // To free the 256MB used
+
+ Packed64 p64 = new Packed64(INDEX, BITS);
+ p64.set(INDEX-1, 1);
+ assertEquals("The value at position " + (INDEX-1)
+ + " should be correct for Packed64", 1, p64.get(INDEX-1));
-}
+ }
+}
Index: lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java Mon Jul 12 14:43:22 CEST 2010
+++ lucene/src/java/org/apache/lucene/util/packed/IdentityReader.java Mon Jul 12 14:43:22 CEST 2010
@@ -0,0 +1,25 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Returns the index as the value in {@link #get}. Useful for representing a 1:1
+ * mapping of another structure.
+ */
+public class IdentityReader implements PackedInts.Reader {
+ private int size;
+
+ public IdentityReader(int size) {
+ this.size = size;
+ }
+
+ public long get(int index) {
+ return index;
+ }
+
+ public int getBitsPerValue() {
+ return 0;
+ }
+
+ public int size() {
+ return size;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java Wed Aug 11 22:33:40 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedCache.java Wed Aug 11 22:33:40 CEST 2010
@@ -0,0 +1,193 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.*;
+
+public class ExposedCache implements FieldCache {
+ private final FieldCache fallback;
+
+ private final ArrayList cache = new ArrayList(5);
+
+ private static final ExposedCache exposedCache;
+ static {
+ exposedCache = new ExposedCache(FieldCache.DEFAULT);
+ }
+ public static ExposedCache getInstance() {
+ return exposedCache;
+ }
+
+ public ExposedCache(FieldCache fallback) {
+ this.fallback = fallback;
+ }
+
+ public TermProvider getProvider(
+ IndexReader reader, String groupName, List fieldNames,
+ Comparator comparator, String comparatorID) throws IOException {
+ if (fieldNames.size() == 0) {
+ throw new IllegalArgumentException("There must be at least 1 field name");
+ }
+ List fieldRequests =
+ new ArrayList(fieldNames.size());
+ for (String fieldName: fieldNames) {
+ fieldRequests.add(new ExposedRequest.Field(
+ fieldName, comparator, comparatorID));
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID, 10, false);
+
+ for (TermProvider provider: cache) {
+ if (provider instanceof GroupTermProvider
+ && ((GroupTermProvider) provider).getRequest().equals(groupRequest)
+ && provider.getReaderHash() == reader.hashCode()) {
+ return provider;
+ }
+ }
+
+ IndexReader[] readers;
+ if (reader.getSequentialSubReaders() == null) {
+ readers = new IndexReader[1];
+ readers[0] = reader;
+ } else {
+ readers = reader.getSequentialSubReaders();
+ }
+
+ List fieldProviders =
+ new ArrayList(readers.length * fieldNames.size());
+
+ for (IndexReader sub: readers) {
+ for (ExposedRequest.Field fieldRequest: fieldRequests) {
+ fieldProviders.add(getProvider(sub, fieldRequest, true, true));
+ }
+ }
+ TermProvider groupProvider = new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ cache.add(groupProvider);
+ return groupProvider;
+ }
+
+ FieldTermProvider getProvider(
+ IndexReader segmentReader, ExposedRequest.Field request,
+ boolean cacheTables, boolean cacheProvider) throws IOException {
+ for (TermProvider provider: cache) {
+ if (provider instanceof FieldTermProvider) {
+ if (provider.getRecursiveHash() == segmentReader.hashCode()
+ && ((FieldTermProvider)provider).getRequest().equals(request)) {
+ return (FieldTermProvider)provider;
+ }
+ }
+ }
+ FieldTermProvider provider =
+ new FieldTermProvider(segmentReader, request, cacheTables);
+ if (cacheProvider) {
+ cache.add(provider);
+ }
+ return provider;
+ }
+
+ public void purgeAllCaches() {
+ fallback.purgeAllCaches();
+ cache.clear();
+ }
+
+ /**
+ * Purges all entries with connections to the given index reader. For exposed
+ * structures that means all structures except the ones that are
+ * FieldTermProviders not relying on the given index reader.
+ * @param r the reader to purge.
+ */
+ public synchronized void purge(IndexReader r) {
+ fallback.purge(r);
+ Iterator remover =
+ cache.iterator();
+ while (remover.hasNext()) {
+ TermProvider provider = remover.next();
+ if (!(provider instanceof FieldTermProvider) ||
+ provider.getRecursiveHash() == r.hashCode()) {
+ remover.remove();
+ }
+ }
+ }
+
+ /* Direct delegations to the default cache */
+
+ public byte[] getBytes(IndexReader reader, String field) throws IOException {
+ return fallback.getBytes(reader, field);
+ }
+
+ public byte[] getBytes(IndexReader reader, String field, ByteParser parser) throws IOException {
+ return fallback.getBytes(reader, field, parser);
+ }
+
+ public short[] getShorts(IndexReader reader, String field) throws IOException {
+ return fallback.getShorts(reader, field);
+ }
+
+ public short[] getShorts(IndexReader reader, String field, ShortParser parser) throws IOException {
+ return fallback.getShorts(reader, field, parser);
+ }
+
+ public int[] getInts(IndexReader reader, String field) throws IOException {
+ return fallback.getInts(reader, field);
+ }
+
+ public int[] getInts(IndexReader reader, String field, IntParser parser) throws IOException {
+ return fallback.getInts(reader, field, parser);
+ }
+
+ public float[] getFloats(IndexReader reader, String field) throws IOException {
+ return fallback.getFloats(reader, field);
+ }
+
+ public float[] getFloats(IndexReader reader, String field, FloatParser parser) throws IOException {
+ return fallback.getFloats(reader, field, parser);
+ }
+
+ public long[] getLongs(IndexReader reader, String field) throws IOException {
+ return fallback.getLongs(reader, field);
+ }
+
+ public long[] getLongs(IndexReader reader, String field, LongParser parser) throws IOException {
+ return fallback.getLongs(reader, field, parser);
+ }
+
+ public double[] getDoubles(IndexReader reader, String field) throws IOException {
+ return fallback.getDoubles(reader, field);
+ }
+
+ public double[] getDoubles(IndexReader reader, String field, DoubleParser parser) throws IOException {
+ return fallback.getDoubles(reader, field, parser);
+ }
+
+ public DocTerms getTerms(IndexReader reader, String field) throws IOException {
+ return fallback.getTerms(reader, field);
+ }
+
+ public DocTerms getTerms(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return fallback.getTerms(reader, field, fasterButMoreRAM);
+ }
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field) throws IOException {
+ return fallback.getTermsIndex(reader, field);
+ }
+
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
+ return fallback.getTermsIndex(reader, field, fasterButMoreRAM);
+ }
+
+ public FieldCache.CacheEntry[] getCacheEntries() {
+ return fallback.getCacheEntries();
+ }
+
+ public void setInfoStream(PrintStream stream) {
+ fallback.setInfoStream(stream);
+ }
+
+ public PrintStream getInfoStream() {
+ return fallback.getInfoStream();
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java Thu Sep 02 11:56:36 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedFieldComparatorSource.java Thu Sep 02 11:56:36 CEST 2010
@@ -0,0 +1,189 @@
+/* $Id:$
+ *
+ * The Summa project.
+ * Copyright (C) 2005-2010 The State and University Library
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+/**
+ * Custom sorter that uses the Exposed framework. Trade-offs are slow first-time
+ * use, fast subsequent use and low memory footprint.
+ */
+public class ExposedFieldComparatorSource extends FieldComparatorSource {
+ private final IndexReader reader;
+ private final Comparator comparator;
+ private final String comparatorID;
+ private final boolean sortNullFirst;
+
+ // TODO: Figure out how to avoid the need for re-creation upon re-open
+ public ExposedFieldComparatorSource(
+ IndexReader reader, Comparator comparator, String comparatorID,
+ boolean sortNullFirst) {
+ this.reader = reader;
+ this.comparatorID = comparatorID;
+ this.comparator = comparator;
+ this.sortNullFirst = sortNullFirst;
+ }
+
+ public ExposedFieldComparatorSource(IndexReader reader, Locale locale) {
+ this(reader,
+ ExposedComparators.collatorToBytesRef(
+ locale == null ? null : Collator.getInstance(locale)),
+ locale == null ? ExposedRequest.LUCENE_ORDER : locale.toString(),
+ false);
+ }
+
+ /**
+ * @param fieldname the field to sort on. If a group of fields is wanted,
+ * field names must be separated by {@code ;}.
+ * Example: {@code foo;bar;zoo}. Note that grouping treats the terms
+ * from the separate fields at the same level and that there should
+ * only be a single term i total for the given fields for a single
+ * document. Grouping is not normally used at this level.
+ * @param numHits the number of hits to return.
+ * @param sortPos ?
+ * @param reversed reversed sort order.
+ * @return a comparator based on the given parameters.
+ * @throws IOException if the
+ */
+ @Override
+ public FieldComparator newComparator(
+ String fieldname, int numHits, int sortPos, boolean reversed)
+ throws IOException {
+ List fieldNames = Arrays.asList(fieldname.split(":"));
+ return new ExposedFieldComparator(
+ reader, fieldname, fieldNames, numHits, sortPos, reversed);
+ }
+
+ private class ExposedFieldComparator extends FieldComparator {
+ private final TermProvider provider;
+
+ private final int undefinedTerm = -1;
+ private final PackedInts.Reader docOrder;
+
+ private final String groupName;
+ private final List fieldNames;
+ private final int numHits;
+ private final int sortPos;
+ private final boolean reversed;
+ private final int factor; // Reverse-indicator
+ private int docBase = 0; // Added to all incoming docIDs
+
+ private int[] order; // docOrder
+ private int bottom; // docOrder
+
+ public ExposedFieldComparator(
+ IndexReader reader, String groupName, List fieldNames,
+ int numHits, int sortPos, boolean reversed) throws IOException {
+ this.groupName = groupName;
+ this.fieldNames = fieldNames;
+ this.provider = ExposedCache.getInstance().getProvider(
+ reader, groupName, fieldNames, comparator, comparatorID);
+
+ this.numHits = numHits;
+ this.sortPos = sortPos;
+ this.reversed = reversed;
+ factor = reversed ? -1 : 1;
+ docOrder = provider.getDocToSingleIndirect();
+
+ order = new int[numHits];
+ }
+
+ @Override
+ public int compare(int slot1, int slot2) {
+ if (!sortNullFirst) {
+ int slot1order = order[slot1];
+ int slot2order = order[slot2];
+ if (slot1order == undefinedTerm) {
+ return slot2order == undefinedTerm ? 0 : -1;
+ } else if (slot2order == undefinedTerm) {
+ return 1;
+ }
+ return factor * (slot1order - slot2order);
+ }
+ // No check for null as null-values are always assigned -1
+ return factor * (order[slot1] - order[slot2]);
+ }
+
+ @Override
+ public void setBottom(int slot) {
+ bottom = order[slot];
+ }
+
+ @Override
+ public int compareBottom(int doc) throws IOException {
+ if (!sortNullFirst) {
+ long bottomOrder = bottom;
+ long docOrderR = docOrder.get(doc+docBase);
+ if (bottomOrder == undefinedTerm) {
+ return docOrderR == undefinedTerm ? 0 : -1;
+ } else if (docOrderR == undefinedTerm) {
+ return 1;
+ }
+ return (int)(factor * (bottomOrder - docOrderR));
+ }
+ // No check for null as null-values are always assigned -1
+ return (int)(factor * (bottom - docOrder.get(doc+docBase)));
+ }
+
+ @Override
+ public void copy(int slot, int doc) throws IOException {
+ // TODO: Remove this
+// System.out.println("Copy called: order[" + slot + "] = "
+// + doc + "+" + docBase + " = " + (doc + docBase));
+// System.out.println("docID " + (doc+docBase) + " has term " + provider.getOrderedTerm(doc+docBase).utf8ToString());
+ order[slot] = (int)docOrder.get(doc+docBase);
+ }
+
+ @Override
+ public void setNextReader(IndexReader reader, int docBase)
+ throws IOException {
+ this.docBase = docBase;
+ }
+
+ private final BytesRef EMPTY = new BytesRef("");
+ @Override
+ public Comparable value(int slot) {
+ try { // A bit cryptic as we need to handle the case of no sort term
+ final long resolvedDocOrder = order[slot];
+ // TODO: Remove this
+/* System.out.println("Resolving docID " + slot + " with docOrder entry "
+ + resolvedDocOrder + " to term "
+ + (resolvedDocOrder == undefinedTerm ? "null" :reader.getTermText(
+ (int)termOrder.get((int)resolvedDocOrder))));
+ */
+ return resolvedDocOrder == undefinedTerm ? EMPTY :
+ provider.getOrderedTerm(resolvedDocOrder);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while extracting term String", e);
+ }
+ }
+ }
+
+
+}
Index: lucene/src/java/org/apache/lucene/util/packed/Packed32.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/Packed32.java (revision 931327)
+++ lucene/src/java/org/apache/lucene/util/packed/Packed32.java Tue Aug 31 10:17:31 CEST 2010
@@ -186,7 +186,7 @@
* @return the value at the given index.
*/
public long get(final int index) {
- final long majorBitPos = index * bitsPerValue;
+ final long majorBitPos = (long)index * bitsPerValue;
final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
@@ -198,7 +198,7 @@
public void set(final int index, final long value) {
final int intValue = (int)value;
- final long majorBitPos = index * bitsPerValue;
+ final long majorBitPos = (long)index * bitsPerValue;
final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
final int base = bitPos * FAC_BITPOS;
Index: lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java Thu Aug 12 10:13:24 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermProviderImpl.java Thu Aug 12 10:13:24 CEST 2010
@@ -0,0 +1,59 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.packed.GrowingMutable;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Default implementation of some methods from TermProvider.
+ */
+public abstract class TermProviderImpl implements TermProvider {
+ protected boolean cacheTables;
+ private PackedInts.Mutable docToSingle = null;
+
+ /**
+ *
+ * @param cacheTables if true, tables such as orderedOrdinals, docID2indirect
+ * and similar should be cached for re-requests after generation.
+ */
+ protected TermProviderImpl(boolean cacheTables) {
+ this.cacheTables = cacheTables;
+ }
+
+ public synchronized PackedInts.Reader getDocToSingleIndirect()
+ throws IOException {
+ if (docToSingle != null) {
+ return docToSingle;
+ }
+ if (getMaxDoc() > Integer.MAX_VALUE) {
+ throw new UnsupportedOperationException(
+ "Unable to handle more than Integer.MAX_VALUE documents. " +
+ "Got macDocs " + getMaxDoc());
+ }
+ // TODO: getMaxDoc() is seriously wonky for GroupTermProvider. What breaks?
+ long sortTime;
+ // TODO: Check why it is extremely slow to start with low maxValue
+ docToSingle = new GrowingMutable(
+ 0, (int)getMaxDoc(), -1, getOrdinalTermCount());
+ try {
+ Iterator ei = getIterator(true);
+ sortTime = System.currentTimeMillis();
+ while (ei.hasNext()) {
+ ExposedTuple tuple = ei.next();
+ docToSingle.set((int)tuple.docID, tuple.indirect);
+/* if (tuple.indirect << 60 == 0) {
+ System.out.print(".");
+ }*/
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to create doc to indirect map", e);
+ }
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println(this.getClass().getSimpleName() + " merge-mapped "
+ + (getMaxDoc()-1) + " docs to single terms in " + sortTime + " ms: " +
+ (sortTime == 0 ? "N/A" : docToSingle.size() / sortTime) + " docs/ms");
+ return docToSingle;
+ }
+}
Index: lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java Fri Aug 13 15:20:51 CEST 2010
+++ lucene/src/java/org/apache/lucene/util/packed/GrowingMutable.java Fri Aug 13 15:20:51 CEST 2010
@@ -0,0 +1,198 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/**
+ * A PackedInts that grows to accommodate any legal data that is added.
+ * Deltas for both array length and values are used so memory-representation
+ * is fairly compact.
+ *
+ * The initial default value for a growing mutable is the given minValue or 0
+ * if no minValue is specified. If a value smaller than minValue is added after
+ * construction, the new default value will be that value and the old non-set
+ * values will still be the old default value. To put it in code: {@code
+ GrowingMutable g = new GrowingMutable(0, 5, 5, 10);
+ assert(g.get(0) == 5);
+ g.set(2, 4);
+ assert(g.get(0) == 5);
+ assert(g.get(1) == 4);
+ }. To avoid error it is imperative to either specify a minValue that is
+ guaranteed to be the minimum value for the life of the mutable or to ensure
+ that all values are assigned explicitly.
+ *
+ * The penalty for using this implementation compared to a plain PackedInts is
+ * slower {@link #set}s as it needs to check boundaries and potentially expand
+ * the internal array. There are no conditionals and very little extra overhead
+ * for {@link #get}s so access is still fast.
+ */
+// TODO: Optimize the speed of array length growth
+public class GrowingMutable implements PackedInts.Mutable {
+ // Grow array length with this factor beyond the immediate need
+ private static final double GROWTH_FACTOR = 1.2;
+
+ private int indexMin = 0;
+ private int indexMax = 0;
+ private long valueMin = 0;
+ private long valueMax = 0;
+ private int size = 0; // Size is explicit to handle grow seamlessly
+
+ private PackedInts.Mutable values;
+
+ /**
+ * Create an empty array of size 0. If it is possible to estimate the amount
+ * and rages of the values to add to the array, it is recommended to use
+ * {@link #GrowingMutable(int, int, long, long)} instead.
+ */
+ public GrowingMutable() {
+ values = PackedInts.getMutable(0, 1);
+ }
+
+ /**
+ * Create an empty array with the given boundaries. The boundaries will be
+ * expanded as needed. The default value is valueMin.
+ * @param indexMin the minimum index for the array.
+ * @param indexMax the maximum index for the array.
+ * @param valueMin the minimum value for the array.
+ * Note that this can be negative.
+ * @param valueMax the maximum value for the array. This must be higher than
+ * or equal to valueMin.
+ * Note: This can be negative.
+ */
+ public GrowingMutable(int indexMin, int indexMax,
+ long valueMin, long valueMax) {
+ if (indexMin > indexMax) {
+ throw new IllegalArgumentException(String.format(
+ "indexMin(%d) must be <= indexMax(%d)", indexMin, indexMax));
+ }
+ if (valueMin > valueMax) {
+ throw new IllegalArgumentException(String.format(
+ "valueMin(%d) must be <= valueMax(%d)", valueMin, valueMax));
+ }
+ this.indexMin = indexMin;
+ this.indexMax = indexMax;
+ this.valueMin = valueMin;
+ this.valueMax = valueMax;
+ this.size = indexMax - indexMin + 1;
+ values = PackedInts.getMutable(
+ indexMax - indexMin + 1, PackedInts.bitsRequired(valueMax - valueMin));
+ }
+
+ // Make sure that there is room. Note: This might adjust the boundaries.
+ private synchronized void checkValues(final int index, final long value) {
+ if (index >= indexMin && index <= indexMax
+ && value >= valueMin && value <= valueMax) {
+ return;
+ }
+ // TODO: Just adjust max is within bitsPerValue
+ if (size() == 0) { // First addition
+ indexMin = index;
+ indexMax = index;
+ valueMin = value;
+ valueMax = value;
+ GrowingMutable newGrowingMutable =
+ new GrowingMutable(indexMin, indexMax, valueMin, valueMax);
+ size = 1;
+ values = newGrowingMutable.values;
+ return;
+ }
+
+ // indexes are grown with factor, values without
+ int newIndexMin =
+ index >= indexMin ? indexMin :
+ indexMax - (int)Math.ceil((indexMax - index) * GROWTH_FACTOR);
+ int newIndexMax =
+ index <= indexMax ? indexMax :
+ indexMin + (int)Math.ceil((index - indexMin) * GROWTH_FACTOR);
+ long newValueMin = Math.min(value, valueMin);
+ long newValueMax = Math.max(value, valueMax);
+
+ GrowingMutable newGrowingMutable =
+ new GrowingMutable(newIndexMin, newIndexMax, newValueMin, newValueMax);
+ for (int i = indexMin ; i <= indexMax ; i++) {
+ newGrowingMutable.set(i, get(i));
+ }
+
+ values = newGrowingMutable.values;
+ indexMin = newIndexMin;
+ indexMax = newIndexMax;
+ valueMin = newValueMin;
+ valueMax = newValueMax;
+ size = indexMax - indexMin; // Size is always minimum to accommodate index
+ }
+
+ public void set(int index, long value) {
+ checkValues(index, value);
+ values.set(index - indexMin, value - valueMin);
+ }
+
+ @Override
+ public String toString() {
+ final int DUMP_SIZE = 10;
+ StringBuffer sb = new StringBuffer(DUMP_SIZE * 10);
+ sb.append("GrowingMutable(");
+ for (int i = indexMin ; i < indexMin + Math.min(size(), DUMP_SIZE) ; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append("i(").append(Integer.toString(i)).append(")=");
+ sb.append(Long.toString(get(i)));
+ }
+ if (size > DUMP_SIZE) {
+ sb.append(", ...");
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ /**
+ * Clear only resets the array. It does not free memory or adjust size.
+ */
+ public void clear() {
+ values.clear();
+ }
+
+ public long get(int index) {
+ return values.get(index - indexMin) + valueMin;
+ }
+
+ public int getBitsPerValue() {
+ return values.getBitsPerValue();
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public int getIndexMin() {
+ return indexMin;
+ }
+
+ public int getIndexMax() {
+ return indexMax;
+ }
+
+ public long getValueMin() {
+ return valueMin;
+ }
+
+ public long getValueMax() {
+ return valueMax;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java Thu Sep 02 11:40:22 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedComparators.java Thu Sep 02 11:40:22 CEST 2010
@@ -0,0 +1,210 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.Comparator;
+
+/**
+ * Wraps different types of comparators.
+ */
+public class ExposedComparators {
+
+ /**
+ * Simple atomic specialization of Comparator. Normally used to
+ * compare terms by treating the arguments as ordinals and performing lookup
+ * in an underlying reader.
+ */
+ public interface OrdinalComparator {
+ int compare(int value1, int value2);
+ }
+
+ public static Comparator collatorToBytesRef(
+ final Collator collator) {
+ return collator == null ? new NaturalComparator()
+ : new BytesRefWrappedCollator(collator);
+ }
+ public static final class BytesRefWrappedCollator implements
+ Comparator {
+ private final Collator collator;
+ public BytesRefWrappedCollator(Collator collator) {
+ this.collator = collator;
+ }
+ public int compare(BytesRef o1, BytesRef o2) {
+ return collator.compare(o1.utf8ToString(), o2.utf8ToString());
+ }
+ public Collator getCollator() {
+ return collator;
+ }
+ }
+ public static final class NaturalComparator implements Comparator {
+ public int compare(BytesRef o1, BytesRef o2) {
+ return o1.compareTo(o2); // TODO: Consider null-case
+ }
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the BytesRefs values and feeding them to the comparator.
+ */
+ public static OrdinalComparator wrap(TermProvider provider,
+ Comparator comparator) {
+ return new BytesRefWrapper(provider, comparator);
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param map indirect map from logical index to term.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the BytesRefs values and feeding them to the comparator.
+ * The BytesRefs are requested by calling getTerm with map[value].
+ */
+ public static OrdinalComparator wrapIndirect(
+ TermProvider provider, int[] map, Comparator comparator) {
+ return new IndirectBytesRefWrapper(provider, map, comparator);
+ }
+
+ /**
+ * @param provider the provider of BytesRefs.
+ * @param map indirect map from logical index to ordinal.
+ * @return a comparator that compares ordinals resolvable by provider by
+ * looking up the CollatorKeys and comparing them.
+ * The keys are requested by calling get with map[value].
+ */
+ public static OrdinalComparator wrapIndirect(
+ CachedCollatorKeyProvider provider, int[] map) {
+ return new IndirectCollatorKeyWrapper(provider, map);
+ }
+
+ /**
+ * Creates a comparator that relies on a list of ExposedTuples to get the
+ * BytesRef for comparison.
+ * @param backingTuples the tuples with the terms to use for comparisons.
+ * @param comparator compares two BytesRef against each other.
+ * @return a comparator that compares indexes relative to the backingTuples.
+ * The BytesRefs are requested by calling {@code backingTuples[index].term}.
+ */
+ public static OrdinalComparator wrapBacking(
+ ExposedTuple[] backingTuples, Comparator comparator) {
+ return new BackingTupleWrapper(backingTuples, comparator);
+ }
+
+ public static OrdinalComparator wrapBacking(CollationKey[] backingKeys) {
+ return new BackingCollatorWrapper(backingKeys);
+ }
+
+ private static class BytesRefWrapper implements OrdinalComparator {
+ private final TermProvider provider;
+ private final Comparator comparator;
+
+ public BytesRefWrapper(
+ TermProvider provider, Comparator comparator) {
+ this.comparator = comparator;
+ this.provider = provider;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ try {
+ return comparator == null ?
+ provider.getTerm(value1).compareTo(provider.getTerm(value2)) :
+ comparator.compare(
+ provider.getTerm(value1), provider.getTerm(value2));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing terms for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class IndirectBytesRefWrapper implements OrdinalComparator {
+ private final TermProvider provider;
+ private final int[] map;
+ private final Comparator comparator;
+
+ public IndirectBytesRefWrapper(
+ TermProvider provider, int[] map, Comparator comparator) {
+ this.provider = provider;
+ this.map = map;
+ this.comparator = comparator;
+ }
+
+ public final int compare(final int value1, final int value2) {
+ try {
+ return comparator.compare(
+ provider.getTerm(map[value1]), provider.getTerm(map[value2]));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing terms for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class IndirectCollatorKeyWrapper implements OrdinalComparator {
+ private final CachedCollatorKeyProvider provider;
+ private final int[] map;
+
+ public IndirectCollatorKeyWrapper(
+ CachedCollatorKeyProvider provider, int[] map) {
+ this.provider = provider;
+ this.map = map;
+ }
+
+ // TODO: Consider null as value
+ public final int compare(final int value1, final int value2) {
+ try {
+ return provider.get(map[value1]).compareTo(provider.get(map[value2]));
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException encountered while comparing keys for the ordinals "
+ + value1 + " and " + value2, e);
+ }
+ }
+ }
+
+ private static class BackingTupleWrapper implements OrdinalComparator {
+ public ExposedTuple[] backingTerms; // Updated from the outside
+ private final Comparator comparator;
+
+ private BackingTupleWrapper(
+ ExposedTuple[] backingTerms, Comparator comparator) {
+ this.backingTerms = backingTerms;
+ this.comparator = comparator;
+ }
+
+ public int compare(int v1, int v2) {
+ try {
+ return comparator == null ?
+ backingTerms[v1].term.compareTo(backingTerms[v2].term) :
+ comparator.compare(backingTerms[v1].term, backingTerms[v2].term);
+ } catch (NullPointerException e) {
+ System.out.println("ddd");
+ return 0;
+ }
+ }
+ }
+
+ private static class BackingCollatorWrapper implements OrdinalComparator {
+ public CollationKey[] backingKeys; // Updated from the outside
+
+ public BackingCollatorWrapper(CollationKey[] backingKeys) {
+ this.backingKeys = backingKeys;
+ }
+
+ public int compare(int v1, int v2) {
+ try {
+ return backingKeys[v1].compareTo(backingKeys[v2]);
+ } catch (NullPointerException e) {
+ return 0;
+ }
+ }
+
+ }
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java Fri Aug 13 14:41:38 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/CachedTermProvider.java Fri Aug 13 14:41:38 CEST 2010
@@ -0,0 +1,120 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Wrapper for TermProvider that provides a flexible cache for terms.
+ * Warning: This cache does not work well for docIDs.
+ *
+ * For random access, a plain setup with no read-ahead is suitable.
+ *
+ * For straight iteration in ordinal order, a setup with maximum read-ahead
+ * is preferable. This is uncommon.
+ *
+ * For iteration in sorted ordinal order, where the ordinal order is fairly
+ * aligned to unicode sorting order, a setup with some read-ahead works well.
+ * This is a common case.
+ *
+ * For merging chunks (See the SegmentReader.getSortedTermOrdinals), where
+ * the ordinal order inside the chunks if fairly aligned to unicode sorting
+ * order, a read-ahead works iff {@link #onlyReadAheadIfSpace} is true as this
+ * prevents values from the beginning of other chunks from being flushed when
+ * values from the current chunk is filled with read-ahead. This requires that
+ * the merger removes processed values from the cache explicitly.
+ *
+ * If the underlying order is used by calling {@link #getOrderedTerm(long)},
+ * {@link #getOrderedField(long)} or {@link #getOrderedOrdinals()}, the
+ * methods are delegated directly to the backing TermProvider. It is the
+ * responsibility of the caller to endure that proper caching of the order
+ * is done at the source level.
+ */
+public class CachedTermProvider extends CachedProvider
+ implements TermProvider {
+ private TermProvider source;
+
+ /**
+ *
+ * @param source the backing term provider.
+ * @param cacheSize the maximum number of elements to hold in cache.
+ * @param readAhead the maximum number of lookups that can be performed
+ * after a plain lookup.
+ * @throws java.io.IOException if the cache could access the source.
+ */
+ public CachedTermProvider(
+ TermProvider source, int cacheSize, int readAhead) throws IOException {
+ super(cacheSize, readAhead, source.getOrdinalTermCount()-1);
+ this.source = source;
+ }
+
+ @Override
+ protected BytesRef lookup(final long index) throws IOException {
+ return source.getTerm(index);
+ }
+
+ public BytesRef getTerm(final long ordinal) throws IOException {
+ return get(ordinal);
+ }
+
+ public Iterator getIterator(boolean collectDocIDs)
+ throws IOException {
+ throw new UnsupportedOperationException(
+ "The cache does not support the creation of iterators");
+ }
+
+ /* Delegations */
+
+ public String getField(long ordinal) throws IOException {
+ return source.getField(ordinal);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return source.getOrderedField(indirect);
+ }
+
+ public BytesRef getOrderedTerm(long indirect) throws IOException {
+ return source.getOrderedTerm(indirect);
+ }
+
+ public long getUniqueTermCount() throws IOException {
+ return source.getUniqueTermCount();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return source.getOrdinalTermCount();
+ }
+
+ public long getMaxDoc() {
+ return source.getMaxDoc();
+ }
+
+ public IndexReader getReader() throws IOException {
+ return source.getReader();
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ return source.getDocsEnum(ordinal, reuse);
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ return source.getOrderedOrdinals();
+ }
+
+ public PackedInts.Reader getDocToSingleIndirect() throws IOException {
+ return source.getDocToSingleIndirect();
+ }
+
+ public int getReaderHash() {
+ return source.getReaderHash();
+ }
+
+ public int getRecursiveHash() {
+ return source.getRecursiveHash();
+ }
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java Thu Sep 02 13:32:47 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/ExposedPOC.java Thu Sep 02 13:32:47 CEST 2010
@@ -0,0 +1,269 @@
+package org.apache.lucene.search.exposed.poc;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.search.exposed.ExposedFieldComparatorSource;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Locale;
+
+public class ExposedPOC {
+ private static final double MAX_HITS = 20;
+
+ public static void main(String[] args)
+ throws IOException, InterruptedException, ParseException {
+ if (args.length == 2 && "optimize".equals(args[0])) {
+ optimize(new File(args[1]));
+ return;
+ }
+ if (args.length != 5) {
+ System.err.println("Need 5 arguments, got " + args.length + "\n");
+ usage();
+ return;
+ }
+ String method = args[0];
+ File location = new File(args[1]);
+ String field = args[2];
+ Locale locale =
+ "null".equalsIgnoreCase(args[3]) ? null : new Locale(args[3]);
+ String defaultField = args[4];
+
+ try {
+ shell(method, location, field, locale, defaultField);
+ } catch (Exception e) {
+ //noinspection CallToPrintStackTrace
+ e.printStackTrace();
+ usage();
+ }
+ }
+
+ private static void optimize(File location) throws IOException {
+ System.out.println("Optimizing " + location + "...");
+ long startTimeOptimize = System.nanoTime();
+ IndexWriter writer = new IndexWriter(FSDirectory.open(location),
+ new IndexWriterConfig(Version.LUCENE_31, new MockAnalyzer()));
+ writer.optimize(1);
+ System.out.println("Optimized index in " + nsToString(
+ System.nanoTime() - startTimeOptimize));
+ writer.close(true);
+ }
+
+ private static void shell(
+ String method, File location, String field, Locale locale,
+ String defaultField)
+ throws IOException, InterruptedException, ParseException {
+ System.out.println(String.format(
+ "Testing sorted search for index at '%s' with sort on field %s with " +
+ "locale %s, using sort-method %s. Heap: %s",
+ location, field, locale, method, getHeap()));
+
+ IndexReader reader = IndexReader.open(FSDirectory.open(location), true);
+ System.out.println(String.format(
+ "Opened index of size %s from %s. the indes has %d documents and %s" +
+ " deletions. Heap: %s",
+ readableSize(calculateSize(location)), location,
+ reader.maxDoc(),
+ reader.hasDeletions() ? "some" : " no",
+ getHeap()));
+
+/* System.out.println(String.format(
+ "Creating %s Sort for field %s with locale %s... Heap: %s",
+ method, field, locale, getHeap()));
+ */
+ long startTimeSort = System.nanoTime();
+ Sort sort;
+ if ("exposed".equals(method) || "expose".equals(method)) {
+ ExposedFieldComparatorSource exposedFCS =
+ new ExposedFieldComparatorSource(reader, locale);
+ sort = new Sort(new SortField(field, exposedFCS));
+ } else if ("default".equals(method)) {
+ sort = locale == null ?
+ new Sort(new SortField(field, SortField.STRING)) :
+ new Sort(new SortField(field, locale));
+ } else {
+ throw new IllegalArgumentException(
+ "The sort method " + method + " is unsupported");
+ }
+ long sortTime = System.nanoTime() - startTimeSort;
+
+ System.out.println(String.format(
+ "Created %s Sort for field %s in %s. Heap: %s",
+ method, field, nsToString(sortTime), getHeap()));
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ System.out.println(String.format(
+ "\nFinished initializing %s structures for field %s.\n"
+ + "Write standard Lucene queries to experiment with sorting speed.\n"
+ + "The StandardAnalyser will be used and the default field is %s.\n"
+ + "Finish with 'EXIT'.", method, field, defaultField));
+ String query;
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, defaultField, new MockAnalyzer());
+
+ boolean first = true;
+ while (true) {
+ if (first) {
+ System.out.print("\nQuery (" + method + " sort, first search might take " +
+ "a while): ");
+ first = false;
+ } else {
+ System.out.print("Query (" + method + " sort): ");
+ }
+ query = br.readLine();
+ if ("".equals(query)) {
+ continue;
+ }
+ if ("EXIT".equals(query)) {
+ break;
+ }
+ try {
+ long startTimeQuery = System.nanoTime();
+ Query q = qp.parse(query);
+ long queryTime = System.nanoTime() - startTimeQuery;
+
+ long startTimeSearch = System.nanoTime();
+ TopFieldDocs topDocs = searcher.search(
+ q.weight(searcher), null, 20, sort, true);
+ long searchTime = System.nanoTime() - startTimeSearch;
+ System.out.println(String.format(
+ "The search for '%s' got %d hits in %s (+ %s for query parsing). "
+ + "Showing %d hits.",
+ query, topDocs.totalHits,
+ nsToString(searchTime),
+ nsToString(queryTime),
+ (int)Math.min(topDocs.totalHits, MAX_HITS)));
+ long startTimeDisplay = System.nanoTime();
+ for (int i = 0 ; i < Math.min(topDocs.totalHits, MAX_HITS) ; i++) {
+ int docID = topDocs.scoreDocs[i].doc;
+ System.out.println(String.format(
+ "Hit #%d was doc #%d with %s:%s",
+ i, docID, field,
+ ((BytesRef)((FieldDoc)topDocs.scoreDocs[i]).fields[0]).
+ utf8ToString()));
+ }
+ System.out.print("Displaying the search result took "
+ + nsToString(
+ System.nanoTime() - startTimeDisplay) + ". ");
+ System.out.println("Heap: " + getHeap());
+ } catch (Exception e) {
+ //noinspection CallToPrintStackTrace
+ e.printStackTrace();
+ }
+ }
+ System.out.println("\nThank you for playing. Please come back.");
+ }
+
+ private static void usage() {
+ System.out.println(
+ "Usage: ExposedPOC exposed|default " +
+ " \n"
+ + "exposed: Uses the expose sorter\n"
+ + "default: Uses the default sorter\n"
+ + ": The location of an optimized Lucene index\n"
+ + ": The field to use for sorting\n"
+ + ": The locale to use for sorting. If null is "
+ + "specified, natural term order is used\n"
+ + ": The field to search when no explicit field is " +
+ "given\n"
+ + "\n"
+ + "Example:\n"
+ + "ExposedPOC expose /mnt/bulk/40GB_index author da freetext"
+ + "\n"
+ + "If the index is is to be optimized, it can be done with\n"
+ + "ExposedPOC optimize \n"
+ + "\n"
+ + "Note: Heap-size is queried after a call to System.gc()\n"
+ + "Note: This is a proof of concept. Expect glitches!"
+ );
+ }
+
+ static String getHeap() throws InterruptedException {
+ String b = "Before GC: " + getHeapDirect();
+ for (int i = 0 ; i < 1 ; i++) {
+ System.gc();
+ Thread.sleep(10);
+ }
+ return b + ", after GC: " + getHeapDirect();
+ }
+
+ private static String getHeapDirect() {
+ return readableSize(Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory()) + "/"
+ + readableSize(Runtime.getRuntime().totalMemory());
+ }
+
+ static String readableSize(long size) {
+ return size > 2 * 1048576 ?
+ size / 1048576 + "MB" :
+ size > 2 * 1024 ?
+ size / 1024 + "KB" :
+ size + "bytes";
+ }
+
+ static long calculateSize(File file) {
+ long size = 0;
+ if (file.isDirectory()) {
+ for (File sub: file.listFiles()) {
+ size += calculateSize(sub);
+ }
+ } else {
+ size += file.length();
+ }
+ return size;
+ }
+
+ static String measureSortTime(final PackedInts.Reader orderedDocs) {
+ Integer[] allDocIDS = new Integer[orderedDocs.size()];
+ for (int i = 0 ; i < allDocIDS.length ; i++) {
+ allDocIDS[i] = i;
+ }
+ long startTimeSort = System.nanoTime();
+ Arrays.sort(allDocIDS, new Comparator() {
+ public int compare(Integer o1, Integer o2) {
+ return (int)(orderedDocs.get(o1) - orderedDocs.get(o2));
+ }
+ });
+ return nsToString(
+ System.nanoTime() - startTimeSort);
+ }
+
+ static long footprint(PackedInts.Reader values) {
+ return values.getBitsPerValue() * values.size() / 8;
+ }
+
+ static String nsToString(long time) {
+ return time > 10L * 1000 * 1000000 ?
+ minutes(time) + " min" :
+// time > 2 * 1000000 ?
+ time / 1000000 + "ms";// :
+// time + "ns";
+ }
+
+ private static String minutes(long num) {
+ long min = num / 60 / 1000 / 1000000;
+ long left = num - (min * 60 * 1000 * 1000000);
+ long sec = left / 1000 / 1000000;
+ String s = Long.toString(sec);
+ while (s.length() < 2) {
+ s = "0" + s;
+ }
+ return min + ":" + s;
+ }
+
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java Wed Aug 11 22:34:24 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedFactory.java Wed Aug 11 22:34:24 CEST 2010
@@ -0,0 +1,81 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+public class ExposedFactory {
+
+ // TODO: Split this so that group is explicit
+ public static TermProvider createProvider(
+ IndexReader reader, String groupName, List fieldNames,
+ Comparator comparator, String comparatorID) throws IOException {
+ if (fieldNames.size() == 0) {
+ throw new IllegalArgumentException("There must be at least 1 field name");
+ }
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() == 1) {
+ // Segment-level single field
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ fieldNames.get(0), comparator, comparatorID);
+ return new FieldTermProvider(reader, request, true);
+ }
+
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() >= 1) {
+ // Segment-level multi field
+ List fieldProviders =
+ new ArrayList(fieldNames.size());
+ List fieldRequests =
+ new ArrayList(fieldNames.size());
+ for (String fieldName: fieldNames) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldName, comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(reader, fieldRequest, false));
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID, 10, false);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+ if (reader.getSequentialSubReaders() == null && fieldNames.size() == 1) {
+ // Index-level single field
+ IndexReader[] subs = reader.getSequentialSubReaders();
+ List fieldProviders =
+ new ArrayList(subs.length);
+ List fieldRequests =
+ new ArrayList(subs.length);
+ for (IndexReader sub: subs) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldNames.get(0), comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(sub, fieldRequest, false));
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID, 10, false);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+ // Index-level multi field
+ IndexReader[] subs = reader.getSequentialSubReaders();
+ List fieldProviders =
+ new ArrayList(subs.length * fieldNames.size());
+ List fieldRequests =
+ new ArrayList(subs.length * fieldNames.size());
+ for (IndexReader sub: subs) {
+ for (String fieldName: fieldNames) {
+ ExposedRequest.Field fieldRequest = new ExposedRequest.Field(
+ fieldName, comparator, comparatorID);
+ fieldRequests.add(fieldRequest);
+ fieldProviders.add(new FieldTermProvider(sub, fieldRequest, false));
+ }
+ }
+ ExposedRequest.Group groupRequest = new ExposedRequest.Group(
+ groupName, fieldRequests, comparator, comparatorID, 10, false);
+ return new GroupTermProvider(
+ reader.hashCode(), fieldProviders, groupRequest, true);
+ }
+}
Index: lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java Thu Aug 12 10:06:46 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestFieldTermProvider.java Thu Aug 12 10:06:46 CEST 2010
@@ -0,0 +1,182 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.text.Collator;
+import java.util.*;
+
+public class TestFieldTermProvider extends LuceneTestCase {
+// new File(System.getProperty("java.io.tmpdir"), "exposed_index");
+// new File("/home/te/projects/lucene/exposed_index");
+// new File("/mnt/bulk/exposed_index");
+ public static final int DOCCOUNT = 10;
+ private ExposedHelper helper;
+
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ helper.close();
+ }
+
+ public void testCreateIndex() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ }
+
+ public void testSegmentcount() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ int subCount = reader.getSequentialSubReaders().length;
+ assertTrue("The number of segments should be >= 2 but was " + subCount,
+ subCount >= 2);
+ reader.close();
+ }
+
+ public void testIndexGeneration() throws Exception {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ long termCount = 0;
+ TermsEnum terms = MultiFields.getFields(reader).
+ terms(ExposedHelper.ID).iterator();
+ while (terms.next() != null) {
+ assertEquals("The ID-term #" + termCount + " should be correct",
+ ExposedHelper.ID_FORMAT.format(termCount),
+ terms.term().utf8ToString());
+ termCount++;
+ }
+
+ assertEquals("There should be the right number of terms",
+ DOCCOUNT, termCount);
+ reader.close();
+ }
+
+ public void testOrdinalAccess() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ TermsEnum terms =
+ reader.getSequentialSubReaders()[0].fields().terms("a").iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ }
+
+ TermProvider segmentProvider = ExposedFactory.createProvider(
+ segment, null, Arrays.asList("a"), null, "null");
+ ExposedRequest.Field request = new ExposedRequest.Field("a", null, "null");
+ ArrayList exposedOrdinals = new ArrayList(DOCCOUNT);
+ for (int i = 0 ; i < segmentProvider.getOrdinalTermCount() ; i++) {
+ exposedOrdinals.add(segmentProvider.getTerm(i).utf8ToString());
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedOrdinals.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedOrdinals.get(i));
+ }
+ reader.close();
+
+ }
+
+ public void testSegmentTermSort() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+ testSegmentTermSort(segment);
+ reader.close();
+ }
+
+ private void testSegmentTermSort(IndexReader segment) throws IOException {
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plainExtraction = new ArrayList(DOCCOUNT);
+ TermsEnum terms = segment.fields().terms("a").iterator();
+ while (terms.next() != null) {
+ plainExtraction.add(terms.term().utf8ToString());
+ System.out.println("Unsorted term #" + (plainExtraction.size() - 1) + ": "
+ + terms.term().utf8ToString());
+ }
+ Collections.sort(plainExtraction, sorter);
+
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(sorter), "collator_da");
+ FieldTermProvider segmentProvider =
+ new FieldTermProvider(segment, request, true);
+
+ ArrayList exposedExtraction = new ArrayList(DOCCOUNT);
+ Iterator ei = segmentProvider.getIterator(false);
+ int count = 0;
+ while (ei.hasNext()) {
+ exposedExtraction.add(ei.next().term.utf8ToString());
+ System.out.println("Exposed sorted term #" + count++ + ": "
+ + exposedExtraction.get(exposedExtraction.size()-1));
+ }
+
+ assertEquals("The two lists of terms should be of equal length",
+ plainExtraction.size(), exposedExtraction.size());
+ for (int i = 0 ; i < plainExtraction.size() ; i++) {
+ assertEquals("The term at index " + i + " should be correct",
+ plainExtraction.get(i), exposedExtraction.get(i));
+ }
+ }
+
+ public void testDocIDMapping() throws IOException {
+ helper.createIndex( DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexReader segment = reader.getSequentialSubReaders()[0];
+ Collator sorter = Collator.getInstance(new Locale("da"));
+
+ ArrayList plain =
+ new ArrayList(DOCCOUNT);
+ for (int docID = 0 ; docID < segment.maxDoc() ; docID++) {
+ plain.add(new ExposedHelper.Pair(
+ docID, segment.document(docID).get("a"), sorter));
+// System.out.println("Plain access added " + plain.get(plain.size()-1));
+ }
+ Collections.sort(plain);
+
+ ExposedRequest.Field request = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(sorter), "collator_da");
+ FieldTermProvider segmentProvider =
+ new FieldTermProvider(segment, request, true);
+
+ ArrayList exposed =
+ new ArrayList(DOCCOUNT);
+ Iterator ei = segmentProvider.getIterator(true);
+
+ while (ei.hasNext()) {
+ ExposedTuple next = ei.next();
+ exposed.add(new ExposedHelper.Pair(
+ next.docID, next.term.utf8ToString(), sorter));
+ }
+ Collections.sort(exposed);
+
+ assertEquals("The two docID->term maps should be of equal length",
+ plain.size(), exposed.size());
+ for (int i = 0 ; i < plain.size() ; i++) {
+ assertEquals("Mapping #" + i + " should be equal but was "
+ + plain.get(i) + " vs. " + exposed.get(i),
+ plain.get(i), exposed.get(i));
+ System.out.println("Sorted docID, term #" + i + ": " + plain.get(i));
+ }
+ reader.close();
+ }
+
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java Wed Aug 11 22:31:32 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermProvider.java Wed Aug 11 22:31:32 CEST 2010
@@ -0,0 +1,105 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Order-oriented variation on {@link org.apache.lucene.index.Terms}.
+ *
+ * Access is primarily through ordinals. This can be done directly with
+ * {@link #getTerm} and {@link #getField} or indirectly through
+ * {@link #getOrderedTerm} and {@link #getOrderedField}. Using the ordered
+ * methods is equivalent to {@code getTerm(getOrder().get(orderPosition))}.
+ */
+public interface TermProvider {
+
+ String getField(long ordinal) throws IOException;
+
+ BytesRef getTerm(long ordinal) throws IOException;
+
+ String getOrderedField(long indirect) throws IOException;
+
+ // Note: indirect might be -1 which should return null
+ BytesRef getOrderedTerm(long indirect) throws IOException;
+
+ long getUniqueTermCount() throws IOException;
+
+ long getOrdinalTermCount() throws IOException;
+
+ long getMaxDoc();
+
+ IndexReader getReader() throws IOException;
+
+ int getReaderHash();
+
+ /**
+ * If the provider is a leaf, the hash-value is that of the IndexReader.
+ * If the provider is a node, the value is the sum of all contained
+ * term-providers hash-values.
+ *
+ * This is useful for mapping segment-level docIDs to index-level then a
+ * group consists of more than one segment.
+ * @return a hash-value useful for determining reader-equivalence.
+ */
+ int getRecursiveHash();
+
+ /**
+ * The ordinals sorted by a previously provided Comparator (typically a
+ * Collator).
+ * @return ordinals ordered by Comparator..
+ * @throws IOException if the ordinals could not be retrieved.
+ */
+ PackedInts.Reader getOrderedOrdinals() throws IOException;
+
+ /**
+ * Mapping from docID to the ordered ordinals returned by
+ * {@link #getOrderedOrdinals()}. Usable for sorting by field value as
+ * comparisons of documents can be done with {@code
+ PackedInts.Reader doc2indirect = getDocToSingleIndirect();
+ ...
+ if (doc2indirect.get(docID1) < doc2indirect.get(docID2)) {
+ System.out.println("Doc #" + docID1 + " comes before doc # " + docID2);
+ }}. If the term for a document is needed it can be retrieved with
+ * {@code getOrderedTerm(doc2indirect.get(docID))}.
+ *
+ * Documents without a corresponding term has the indirect value {@code -1}.
+ * @return a map from document IDs to the indirect values for the terms for
+ * the documents.
+ * @throws IOException if the index could not be accessed,
+ */
+ // TODO: Handle missing value (-1? Max+1? Defined by "empty first"?)
+ PackedInts.Reader getDocToSingleIndirect() throws IOException;
+
+ /**
+ * @param collectDocIDs if true, the document IDs are returned as part of the
+ * tuples. Note that this normally increases the processing time
+ * significantly.
+ * @return an iterator over tuples in the order defined for the TermProvider.
+ * @throws IOException if the iterator could not be constructed.
+ */
+ Iterator getIterator(boolean collectDocIDs) throws IOException;
+
+ DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException;
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java Wed Jul 21 13:59:42 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedTuple.java Wed Jul 21 13:59:42 CEST 2010
@@ -0,0 +1,71 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A representation of Field, Term, ordinal, indirect and potentially docID.
+ */
+public class ExposedTuple {
+ public String field;
+ /**
+ * The term itself. Note that this is re-used by the iterator, so users of
+ * the iterator must finish processing the current term before calling next.
+ */
+ public BytesRef term;
+ /**
+ * The ordinal for the term, used for direct lookup.
+ */
+ public long ordinal;
+ /**
+ * The indirect for the term, used for lookup with
+ * {@link TermProvider#getOrderedTerm(long)}.
+ */
+ public long indirect;
+ /**
+ * A docID for the term or -1 if docIDs are not requested or not existing.
+ * If docIDs are requested and if there are multiple docIDs for the term,
+ * multiple ExposedTuples with the same term and the same ordinal will be
+ * used.
+ */
+ public long docID;
+
+ /*
+ * The order of the term, relative to the other Terms delivered, when using
+ * an Iterator from {@link #getExposedTuples}. Starts at 0, increments by at
+ * most 1 for each subsequent ExposedTuple from the iterator.
+ */
+ //public long order;
+
+ public ExposedTuple(
+ String field, BytesRef term, long ordinal, long indirect, long docID) {
+ this.field = field;
+ this.term = term;
+ this.ordinal = ordinal;
+ this.indirect = indirect;
+ this.docID = docID;
+ }
+
+ public ExposedTuple(ExposedTuple other) {
+ this.field = other.field;
+ this.term = other.term;
+ this.ordinal = other.ordinal;
+ this.indirect = other.indirect;
+ this.docID = other.docID;
+ }
+
+ public String toString() {
+ return "ExposedTuple(" + field + ":"
+ + (term == null ? "null" : term.utf8ToString()) + ", ord=" + ordinal
+ + ", indirect=" + indirect + ", docID=" + docID + ")";
+ }
+
+ // Convenience
+ public void set(
+ String field, BytesRef term, long ordinal, long indirect, long docID) {
+ this.field = field;
+ this.term = term;
+ this.ordinal = ordinal;
+ this.indirect = indirect;
+ this.docID = docID;
+ }
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java Fri Aug 13 15:51:11 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/MergingTermDocIterator.java Fri Aug 13 15:51:11 CEST 2010
@@ -0,0 +1,224 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Multi source Iterator that merges the ExposedTuples according to the given
+ * comparator. Designed to work tightly with {@link GroupTermProvider}.
+ *
+ * It is recommended to use {@link CachedTermProvider} with look-ahead on the
+ * sources or otherwise ensure that ordered iteration is fast if no docIDs are
+ * to be returned. If docIDs are to be returned, caching at the source level
+ * will not help.
+ *
+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless
+ * {@link #setReuseTuple(boolean)} is called with false.
+ *
+ * The merger works by keeping a list of the first elements from all sources
+ * and seeding a priority queue with indexes 0...#sources-1. When the PQ is
+ * popped, the elements corresponding to the indexes in the PQ are compared.
+ * When an element is returned, it is delivered to the iterator. The next time
+ * next() is called on the iterator, the index for the previous tuple is
+ * re-inserted into the PQ, unless the source for that index is empty.
+ * This ensures full reuse of the tuples and thus low impact on GC.
+ */
+class MergingTermDocIterator implements Iterator {
+ private final List> iterators;
+ private final boolean collectDocIDs;
+ private final GroupTermProvider groupProvider;
+
+ private final ExposedTuple[] backingTuples;
+ private final ExposedPriorityQueue pq;
+
+ private final Collator collator;
+ private final CollationKey[] backingKeys; // Only used with Collator
+
+ // TODO: Make reuseTuple = true work
+ private boolean reuseTuple = false;
+ private long indirect = 0;
+
+ /*
+ If pending is true, the currentIndex points to the iterator and backingTuple
+ that is to be drained.
+ */
+ private boolean pending = false; // true == Terms or docIDs are ready in tuple
+ /**
+ * If pending is true, this is ready for delivery.
+ */
+ private final ExposedTuple tuple =
+ new ExposedTuple("", new BytesRef(), -1, -1, -1);
+ /**
+ * If pending is true, this is the index for the source responsible for
+ * delivering the tuple.
+ */
+ private int currentIndex = -1;
+
+ /**
+ * If this is true and the given comparator is a
+ * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized
+ * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the
+ * chunk-sort speed.
+ */
+ public static boolean optimizeCollator = true;
+
+ public MergingTermDocIterator(
+ GroupTermProvider groupProvider, List sources,
+ Comparator comparator, boolean collectDocIDs)
+ throws IOException {
+ this.groupProvider = groupProvider;
+ this.collectDocIDs = collectDocIDs;
+
+ iterators = new ArrayList>(sources.size());
+ backingTuples = new ExposedTuple[sources.size()];
+
+ collator = optimizeCollator &&
+ comparator instanceof ExposedComparators.BytesRefWrappedCollator
+ ? ((ExposedComparators.BytesRefWrappedCollator)comparator).getCollator()
+ : null;
+ backingKeys = new CollationKey[sources.size()];
+
+ ExposedComparators.OrdinalComparator wrappedComparator =
+ collator == null ?
+ ExposedComparators.wrapBacking(backingTuples, comparator)
+ : ExposedComparators.wrapBacking(backingKeys);
+ pq = new ExposedPriorityQueue(wrappedComparator, sources.size());
+ int index = 0;
+ for (TermProvider source: sources) {
+ Iterator iterator = source.getIterator(collectDocIDs);
+ iterators.add(iterator);
+ ExposedTuple tuple = getNextTuple(index);
+ if (tuple != null) {
+ backingTuples[index] = tuple;
+ if (collator != null) {
+ backingKeys[index] =
+ collator.getCollationKey(tuple.term.utf8ToString());
+ }
+ pq.add(index);
+ }
+ index++;
+ }
+ }
+
+ // Ensure that we can deliver an id
+ public boolean hasNext() {
+ while (true) {
+ if (pending) {
+// System.out.println("hasnext ***" + tuple.term.utf8ToString());
+ return true;
+ }
+ if (pq.size() == 0) {
+ return false;
+ }
+ try {
+ seekToNextTuple();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next docID", e);
+ }
+ }
+ }
+
+ public ExposedTuple next() {
+// System.out.println("next " + tuple.term.utf8ToString());
+ if (!hasNext()) {
+ throw new IllegalStateException("The iterator is depleted");
+ }
+ ExposedTuple delivery = reuseTuple ? tuple : new ExposedTuple(tuple);
+ pending = false;
+// System.out.println("Merging delivery: " + delivery);
+ return delivery;
+ }
+
+ // Pops a tuple from the iterator indicated by the index and modifies term
+ // ordinal and docID from segment to index level.
+ // If index == -1 or hasNext() == false, null is returned.
+ private ExposedTuple getNextTuple(int index) throws IOException {
+ if (index == -1 || !iterators.get(index).hasNext()) {
+ return null;
+ }
+ // TODO: Avoid the creation of a new tuple by assigning to existing tuple
+ ExposedTuple nextTuple = new ExposedTuple(iterators.get(index).next());
+ nextTuple.docID = groupProvider.segmentToIndexDocID(
+ index, nextTuple.docID);
+ nextTuple.ordinal = groupProvider.segmentToIndexTermOrdinal(
+ index, nextTuple.ordinal);
+ nextTuple.indirect = -1; // Just for good measure: It will be updated later
+ return nextTuple;
+ }
+
+ private void seekToNextTuple() throws IOException {
+
+/*
+ if (collectDocIDs && currentIndex != -1) {
+
+ // Start by checking if the previous source has more tuples with the
+ // same ordinal (e.g. if it has more docIDs)
+ ExposedTuple candidate = backingTuples[currentIndex];
+ if (candidate != null && candidate.ordinal == tuple.ordinal) {
+ // Field, term, ordinal and indirect are the same
+ tuple.docID = candidate.docID;
+ pending = true;
+ backingTuples[currentIndex] = getNextTuple(currentIndex);
+ return;
+ }
+ }
+ */
+ // DocID-optimization must take place before reinserting into pq
+
+ // Take the next tuple
+ while (!pending && pq.size() > 0) {
+ ExposedTuple newTuple = pop();
+ if (newTuple == null) {
+ break;
+ }
+
+ if (tuple.term.equals(newTuple.term)) {
+ if (!collectDocIDs) {
+ continue; // Skip duplicates whan no docID
+ }
+ tuple.set(newTuple.field, tuple.term, tuple.ordinal, tuple.indirect,
+ newTuple.docID);
+ } else {
+ tuple.set(newTuple.field, newTuple.term, newTuple.ordinal, indirect++,
+ newTuple.docID);
+ }
+ pending = true;
+ }
+ }
+
+ private ExposedTuple pop() throws IOException {
+ currentIndex = pq.pop();
+ if (currentIndex == -1) {
+ return null;
+ }
+ ExposedTuple foundTuple = backingTuples[currentIndex];
+
+ ExposedTuple newTuple = getNextTuple(currentIndex);
+ if (newTuple != null) {
+ backingTuples[currentIndex] = newTuple;
+ if (collator != null) {
+ backingKeys[currentIndex] =
+ collator.getCollationKey(newTuple.term.utf8ToString());
+ }
+ pq.add(currentIndex);
+ }
+ return foundTuple;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("Not a valid operation");
+ }
+
+ public void setReuseTuple(boolean reuseTuple) {
+ this.reuseTuple = reuseTuple;
+ }
+
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java Thu Aug 12 13:57:51 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/poc/MockTokenFilter.java Thu Aug 12 13:57:51 CEST 2010
@@ -0,0 +1,103 @@
+package org.apache.lucene.search.exposed.poc;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty;
+import static org.apache.lucene.util.automaton.BasicAutomata.makeString;
+
+/**
+ * A tokenfilter for testing that removes terms accepted by a DFA.
+ *
+ * - Union a list of singletons to act like a stopfilter.
+ *
- Use the complement to act like a keepwordfilter
+ *
- Use a regex like
.{12,} to act like a lengthfilter
+ *
+ */
+public final class MockTokenFilter extends TokenFilter {
+ /** Empty set of stopwords */
+ public static final CharacterRunAutomaton EMPTY_STOPSET =
+ new CharacterRunAutomaton(makeEmpty());
+
+ /** Set of common english stopwords */
+ public static final CharacterRunAutomaton ENGLISH_STOPSET =
+ new CharacterRunAutomaton(BasicOperations.union(Arrays.asList(
+ makeString("a"), makeString("an"), makeString("and"), makeString("are"),
+ makeString("as"), makeString("at"), makeString("be"), makeString("but"),
+ makeString("by"), makeString("for"), makeString("if"), makeString("in"),
+ makeString("into"), makeString("is"), makeString("it"), makeString("no"),
+ makeString("not"), makeString("of"), makeString("on"), makeString("or"),
+ makeString("such"), makeString("that"), makeString("the"), makeString("their"),
+ makeString("then"), makeString("there"), makeString("these"), makeString("they"),
+ makeString("this"), makeString("to"), makeString("was"), makeString("will"),
+ makeString("with"))));
+
+ private final CharacterRunAutomaton filter;
+ private boolean enablePositionIncrements = false;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ super(input);
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ // return the first non-stop word found
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (!filter.run(termAtt.buffer(), 0, termAtt.length())) {
+ if (enablePositionIncrements) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+ return true;
+ }
+ skippedPositions += posIncrAtt.getPositionIncrement();
+ }
+ // reached EOS -- return false
+ return false;
+ }
+
+ /**
+ * @see #setEnablePositionIncrements(boolean)
+ */
+ public boolean getEnablePositionIncrements() {
+ return enablePositionIncrements;
+ }
+
+ /**
+ * If true, this Filter will preserve
+ * positions of the incoming tokens (ie, accumulate and
+ * set position increments of the removed stop tokens).
+ */
+ public void setEnablePositionIncrements(boolean enable) {
+ this.enablePositionIncrements = enable;
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java Tue Aug 31 16:31:26 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/FieldTermProvider.java Tue Aug 31 16:31:26 CEST 2010
@@ -0,0 +1,451 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.IdentityReader;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Keeps track of a single field in a single segment. Optimized towards
+ * SegmentReaders although it can be used with other IndexReaders as long as
+ * they are leafs (has no sub readers).
+ *
+ * The provider is lazy meaning that the ordering is only calculated when
+ * needed.
+ */
+public class FieldTermProvider extends TermProviderImpl {
+ private final IndexReader reader; // This is treated as a SegmentReader
+ private ExposedRequest.Field request;
+ private PackedInts.Reader order = null;
+
+
+ // TODO: Consider letting the cache be a factor of term count
+ /**
+ * The minimum number of Terms or ExtendedTerms to hold in cache during
+ * processing.
+ * The allocated amount of memory depends on average term size and whether
+ * CollatorKey-based sorting is used. With an average term-length of 20
+ * characters, each cache-entry takes approximately 100 bytes without
+ * CollatorKey and 300 with CollatorKey.
+ *
+ * .
+ * 10.000 is thus roughly equivalent to either 1 or 3 MB of RAM depending on
+ * Collator presence.
+ */
+ static int minSortCacheSize = 20000; // 2-6 MB
+ static double sortCacheFraction = 0.05; // 1M docs = 5-15 MB
+ static int maxSortCacheSize = 300000; // 30-90 MB
+ private int sortCacheSize;
+
+ static int mergeChunkFragmentationFactor = 3; // TODO: Experiment with this
+ static int iteratorCacheSize = 5000;
+ static int iteratorReadAhead = 100;
+
+ /**
+ * If this is true and the given comparator is a
+ * {@link ExposedComparators.BytesRefWrappedCollator}, sorting is optimized
+ * by using CollatorKeys. This requires ~1-3 MB extra memory but doubles the
+ * chunk-sort speed.
+ */
+ public static boolean optimizeCollator = true;
+
+ private Terms terms;
+ private TermsEnum termsEnum;
+
+ public FieldTermProvider(IndexReader reader, ExposedRequest.Field request,
+ boolean cacheTables) throws IOException {
+ super(cacheTables);
+ if (reader.getSequentialSubReaders() != null) {
+ throw new IllegalArgumentException(
+ "The IndexReader should be a leaf (no sub readers). It contained "
+ + reader.getSequentialSubReaders().length + " sub readers");
+ }
+ this.reader = reader;
+ this.request = request;
+
+ terms = reader.fields().terms(request.getField());
+ termsEnum = terms == null ? null : terms.iterator(); // It's okay to be empty
+ sortCacheSize = getSortCacheSize(reader);
+ }
+
+ public ExposedRequest.Field getRequest() {
+ return request;
+ }
+
+ public String getField(long ordinal) {
+ return request.getField();
+ }
+
+ private static final int ITERATE_LIMIT = 10;
+ private long lastOrdinalRequest = -1;
+ public synchronized BytesRef getTerm(final long ordinal) throws IOException {
+ if (termsEnum == null) {
+ throw new IOException("No terms for field " + request.getField()
+ + " in segment " + reader + ". Requested ordinal was " + ordinal);
+ }
+ // TODO: Upstream this simple sequential access optimization
+ if (lastOrdinalRequest <= ordinal &&
+ ordinal <= lastOrdinalRequest + ITERATE_LIMIT) {
+ BytesRef term = termsEnum.term();
+ while (lastOrdinalRequest != ordinal) {
+ term = termsEnum.next();
+ if (term == null) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+ lastOrdinalRequest++;
+ }
+ return new BytesRef(term);
+ }
+
+ if (TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+ BytesRef result = termsEnum.term();
+ lastOrdinalRequest = ordinal;
+ return new BytesRef(result);
+ }
+
+ public DocsEnum getDocsEnum(long ordinal, DocsEnum reuse) throws IOException {
+ if (termsEnum == null) {
+ throw new IOException("No terms or docIDs for field " + request.getField()
+ + " in segment " + reader + ". Requested ordinal was " + ordinal);
+ }
+ if (TermsEnum.SeekStatus.FOUND != termsEnum.seek(ordinal)) {
+ throw new IOException("Unable to locate term for ordinal " + ordinal);
+ }
+ return termsEnum.docs(null, reuse);
+ }
+
+ public String getOrderedField(long indirect) throws IOException {
+ return request.getField();
+ }
+
+ public BytesRef getOrderedTerm(final long indirect) throws IOException {
+ return indirect == -1 ? null :
+ getTerm(getOrderedOrdinals().get((int)indirect));
+ }
+
+ public long getUniqueTermCount() throws IOException {
+ return terms == null ? 0 : terms.getUniqueTermCount();
+ }
+
+ public long getOrdinalTermCount() throws IOException {
+ return terms == null ? 0 : terms.getUniqueTermCount();
+ }
+
+ public long getMaxDoc() {
+ return reader.maxDoc();
+ }
+
+ public IndexReader getReader() throws IOException {
+ return reader;
+ }
+
+ public int getReaderHash() {
+ return reader.hashCode();
+ }
+
+ public int getRecursiveHash() {
+ return reader.hashCode();
+ }
+
+ public PackedInts.Reader getOrderedOrdinals() throws IOException {
+ if (termsEnum == null) {
+ return PackedInts.getMutable(0, 1);
+ }
+
+ if (order != null) {
+ return order;
+ }
+ PackedInts.Reader newOrder;
+ if (ExposedRequest.LUCENE_ORDER.equals(request.getComparatorID())) {
+ newOrder = new IdentityReader((int)terms.getUniqueTermCount());
+ } else {
+ newOrder = sortOrdinals();
+ }
+
+ if (cacheTables) {
+ order = newOrder;
+ }
+ return newOrder;
+ }
+
+ public Iterator getIterator(
+ boolean collectDocIDs) throws IOException {
+ PackedInts.Reader order = getOrderedOrdinals();
+ if (collectDocIDs) {
+ return new TermDocIterator(this, true);
+ }
+ this.order = order;
+ CachedTermProvider cache = new CachedTermProvider(
+ this, iteratorCacheSize, iteratorReadAhead);
+ return new TermDocIterator(cache, false);
+ }
+
+ private PackedInts.Reader sortOrdinals() throws IOException {
+ int termCount = (int)getOrdinalTermCount();
+ int[] ordered = new int[termCount];
+ for (int i = 0 ; i < termCount ; i++) {
+ ordered[i] = i; // Initial order = Lucene
+ }
+
+ long startTime = System.nanoTime();
+ sort(ordered);
+ long sortTime = (System.nanoTime() - startTime);
+
+
+ // TODO: Remove this
+ System.out.println("Chunk total sort for field " + getField(0)
+ + ": " + ExposedUtil.time("terms", ordered.length, sortTime / 1000000));
+/* System.out.println(String.format(
+ "Sorted %d Terms in %s out of which %s (%s%%) was lookups and " +
+ "%s (%s%%) was collation key creation. " +
+ "The cache (%d terms) got %d requests with %d (%s%%) misses",
+ termCount, nsToString(sortTime),
+ nsToString(lookupTime),
+ lookupTime * 100 / sortTime,
+ nsToString(collatorKeyCreation),
+ collatorKeyCreation * 100 / sortTime,
+ sortCacheSize, cacheRequests,
+ cacheMisses, cacheMisses * 100 / cacheRequests));
+ */
+ PackedInts.Mutable packed = PackedInts.getMutable(
+ termCount, PackedInts.bitsRequired(termCount));
+ for (int i = 0 ; i < termCount ; i++) {
+ packed.set(i, ordered[i]); // Save space by offsetting min to 0
+ }
+ return packed;
+ }
+
+ /*
+ * Starts by dividing the ordered array in logical chunks, then sorts each
+ * chunk separately and finishes by merging the chunks.
+ */
+ private void sort(final int[] ordinals) throws IOException {
+ int chunkSize = Math.max(sortCacheSize, ordinals.length / sortCacheSize);
+ int chunkCount = (int) Math.ceil(ordinals.length * 1.0 / chunkSize);
+
+ // We do not thread the sort as the caching of Strings is more important
+ // than processing power.
+
+ // We sort in chunks so the cache is 100% effective
+ if (optimizeCollator && request.getComparator() instanceof
+ ExposedComparators.BytesRefWrappedCollator) {
+// System.out.println("Using CollatorKey optimized chunk sort");
+ optimizedChunkSort(ordinals, chunkSize);
+ } else {
+ chunkSort(ordinals, chunkSize);
+ }
+
+ if (chunkSize >= ordinals.length) {
+ return; // Only one chunk. No need for merging
+ }
+
+ // We have sorted chunks. Commence the merging!
+
+ chunkMerge(ordinals, chunkSize, chunkCount);
+ }
+
+ private void chunkMerge(
+ int[] ordinals, int chunkSize, int chunkCount) throws IOException {
+ // Merging up to cache-size chunks requires an efficient way to determine
+ // the chunk with the lowest value. As locality is not that important with
+ // all comparables in cache, we use a heap.
+ // The heap contains an index (int) for all active chunks in the term order
+ // array. When an index is popped, it is incremented and re-inserted unless
+ // it is a block start index in which case it is just discarded.
+
+ System.out.println("Beginning merge sort of " + ordinals.length
+ + " ordinals in " + chunkCount + " chunks");
+ long mergeTime = System.currentTimeMillis();
+ CachedProvider cache;
+ ExposedComparators.OrdinalComparator indirectComparator;
+ if (optimizeCollator && request.getComparator() instanceof
+ ExposedComparators.BytesRefWrappedCollator) {
+ Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request.
+ getComparator()).getCollator();
+ CachedCollatorKeyProvider keyCache = new CachedCollatorKeyProvider(
+ this, collator, sortCacheSize, chunkSize-1);
+ keyCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount /
+ mergeChunkFragmentationFactor));
+ keyCache.setOnlyReadAheadIfSpace(true);
+ keyCache.setStopReadAheadOnExistingValue(true);
+ indirectComparator = ExposedComparators.wrapIndirect(keyCache, ordinals);
+ cache = keyCache;
+ } else {
+ CachedTermProvider termCache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ // Configure the cache so that read ahead is lower as the access pattern
+ // to the ordinals are not guaranteed linear
+ termCache.setReadAhead(Math.max(100, sortCacheSize / chunkCount /
+ mergeChunkFragmentationFactor));
+ termCache.setOnlyReadAheadIfSpace(true);
+ termCache.setStopReadAheadOnExistingValue(true);
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ termCache.getTerm(i); // Warm cache
+ }
+ indirectComparator = ExposedComparators.wrapIndirect(
+ termCache, ordinals, request.getComparator());
+ cache = termCache;
+ }
+
+ ExposedPriorityQueue pq = new ExposedPriorityQueue(
+ indirectComparator, chunkCount);
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ pq.add(i);
+ }
+
+ int[] sorted = new int[ordinals.length];
+ for (int i = 0 ; i < sorted.length ; i++) {
+ Integer next = pq.pop();
+ if (next == -1) {
+ throw new IllegalStateException(
+ "Popping the heap should never return -1");
+ }
+ sorted[i] = ordinals[next];
+ cache.release(sorted[i]); // Important for cache read ahead efficiency
+ if (++next % chunkSize != 0 && next != ordinals.length) {
+ pq.add(next);
+ }
+ }
+ mergeTime = System.currentTimeMillis() - mergeTime;
+ System.out.println(
+ "Merged " + ExposedUtil.time("chunks", chunkCount, mergeTime)
+ + " aka " + ExposedUtil.time("terms", sorted.length, mergeTime)
+ + ": " + cache.getStats());
+ System.arraycopy(sorted, 0, ordinals, 0, sorted.length);
+/* System.out.println(String.format(
+ "Heap merged %d sorted chunks of size %d (cache: %d, total terms: %s)" +
+ " in %s with %d cache misses (%d combined for both sort passes)",
+ chunkCount, chunkSize, sortCacheSize, ordered.length,
+ nsToString(System.nanoTime() - startTimeHeap),
+ cacheMisses - oldHeapCacheMisses, cacheMisses - oldCacheMisses));*/
+ }
+
+ private void chunkSort(int[] ordinals, int chunkSize) throws IOException {
+ CachedTermProvider cache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ // Sort the chunks individually
+ //long startTimeMerge = System.nanoTime();
+ ExposedComparators.OrdinalComparator comparator =
+ ExposedComparators.wrap(cache, request.getComparator());
+ for (int i = 0 ; i < ordinals.length ; i += chunkSize) {
+ long chunkTime = System.currentTimeMillis();
+ cache.getTerm(i); // Tim-sort starts at 1 so we init the read-ahead at 0
+ ExposedTimSort.sort(
+ ordinals, i, Math.min(i + chunkSize, ordinals.length), comparator);
+ chunkTime = System.currentTimeMillis() - chunkTime;
+ int percent = (Math.min(i + chunkSize, ordinals.length)-1) *100
+ / ordinals.length;
+ System.out.println("Chunk sorted " + percent + "% " + i + "-"
+ + (Math.min(i + chunkSize, ordinals.length)-1) + ": "
+ + ExposedUtil.time("ordinals", Math.min(i + chunkSize,
+ ordinals.length) - i, chunkTime) + ": "+ cache.getStats());
+ cache.clear();
+ }
+ }
+
+ // Ordinals _must_ be monotonously increasing
+ private void optimizedChunkSort(
+ int[] ordinals, int chunkSize) throws IOException {
+ Collator collator = ((ExposedComparators.BytesRefWrappedCollator)request.
+ getComparator()).getCollator();
+ CachedTermProvider cache = new CachedTermProvider(
+ this, sortCacheSize, chunkSize-1);
+ CollatorPair[] keys = new CollatorPair[chunkSize];
+ for (int start = 0 ; start < ordinals.length ; start += chunkSize) {
+ long chunkTime = System.currentTimeMillis();
+
+ int end = Math.min(start + chunkSize, ordinals.length); // Exclusive
+ // Fill
+ for (int index = start ; index < end ; index++) {
+ keys[index - start] = new CollatorPair(
+ ordinals[index], collator.getCollationKey(
+ cache.getTerm(ordinals[index]).utf8ToString()));
+ }
+ // Sort
+ ExposedTimSort.sort(
+ ordinals, start, end, new TimComparator(start, keys));
+/* // Store
+ for (int index = start ; index < end ; index++) {
+ ordinals[index] = (int) keys[index - start].ordinal;
+ }*/
+
+ chunkTime = System.currentTimeMillis() - chunkTime;
+ int percent = (int) ((long)end * 100 / ordinals.length);
+ System.out.println("Chunk sorted " + percent + "% "
+ + start + "-" + (end-1) + ": "
+ + ExposedUtil.time("ordinals", end - start, chunkTime)
+ + ": "+ cache.getStats());
+ cache.clear();
+ }
+ }
+ private static final class TimComparator
+ implements ExposedComparators.OrdinalComparator {
+ private int start; // Inclusive
+ private CollatorPair[] keys;
+
+ public TimComparator(int start, CollatorPair[] keys) {
+ this.start = start;
+ this.keys = keys;
+ }
+
+ public int compare(int value1, int value2) {
+ return keys[value1 - start].compareTo(keys[value2 - start]);
+ }
+ }
+
+
+ private static final class CollatorPair implements Comparable{
+ private long ordinal;
+ private CollationKey key;
+
+ private CollatorPair(long ordinal, CollationKey key) {
+ this.ordinal = ordinal;
+ this.key = key;
+ }
+ public int compareTo(CollatorPair o) {
+ return key.compareTo(o.key);
+ }
+ public long getOrdinal() {
+ return ordinal;
+ }
+ }
+
+ /**
+ * Calculate the sort cache size as a fraction of the total number of
+ * documents in the reader,
+ * @param reader the reader to cache.
+ * @return a cache size that fits in scale to the reader.
+ */
+ private int getSortCacheSize(IndexReader reader) {
+ final int fraction = (int) (reader.maxDoc() * sortCacheFraction);
+ return Math.max(maxSortCacheSize, Math.min(minSortCacheSize, fraction));
+ }
+}
\ No newline at end of file
Index: lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java Thu Sep 02 14:07:15 CEST 2010
+++ lucene/src/test/org/apache/lucene/search/exposed/TestExposedCache.java Thu Sep 02 14:07:15 CEST 2010
@@ -0,0 +1,514 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.*;
+
+
+public class TestExposedCache extends LuceneTestCase {
+ public static final int DOCCOUNT = 10;
+ private ExposedHelper helper;
+ private ExposedCache cache;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ cache = new ExposedCache(FieldCache.DEFAULT);
+ helper = new ExposedHelper();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ cache.purgeAllCaches();
+ //helper.close();
+ }
+
+ public void testPlainSortDump() throws Exception {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.EVEN, new MockAnalyzer());
+ Query q = qp.parse("true");
+ Sort aSort = new Sort(new SortField("a", new Locale("da")));
+
+ TopFieldDocs docs = searcher.search(q, null, 100, aSort);
+ dumpDocs(reader, docs, "a");
+ reader.close();
+ }
+
+ public void testIndirectIndex() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+
+ PackedInts.Reader orderedOrdinals = provider.getOrderedOrdinals();
+ Iterator iterator = provider.getIterator(false);
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+ assertEquals("The provided term should match ordinal lookup term",
+ tuple.term.utf8ToString(),
+ provider.getTerm(tuple.ordinal).utf8ToString());
+ assertEquals("The provided term should match indirect lookup term",
+ tuple.term.utf8ToString(),
+ provider.getOrderedTerm(tuple.indirect).utf8ToString());
+ }
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ String exposed = provider.getOrderedTerm(
+ provider.getDocToSingleIndirect().get(docID)).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }
+ }
+
+ public void testIndirectSegment() throws IOException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0]; // SegmentReader
+
+ ExposedRequest.Field fRequest = new ExposedRequest.Field(
+ "a", ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, fRequest, true, true);
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ long indirect = provider.getDocToSingleIndirect().get(docID);
+ String exposed = indirect == -1 ? null :
+ provider.getOrderedTerm(indirect).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }
+ }
+
+ public void testDocCount() throws IOException {
+ int docs = 12000;
+ helper.createIndex(docs, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ ExposedCache.getInstance().purgeAllCaches();
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ assertEquals("The number of documents should be correct",
+ reader.maxDoc(), provider.getMaxDoc());
+ }
+
+ public void testReopen() throws IOException, ParseException {
+ final int RUNS = 5;
+ final int UPDATE_SIZE = 5000;
+ List exposedSortTimes = new ArrayList(RUNS);
+
+ helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 2);
+
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer());
+ Query query = qp.parse(ExposedHelper.ALL);
+ Sort plainSort = new Sort(new SortField("a", new Locale("da")));
+
+ exposedSortTimes.add(testReopen(reader, query, plainSort));
+ for (int i = 1 ; i < RUNS ; i++) {
+ System.out.println("\nModifying index and re-opening reader "
+ + (i+1) + "/" + RUNS);
+ helper.createIndex(UPDATE_SIZE, Arrays.asList("a", "b"), 20, 1);
+ IndexReader oldReader = reader;
+ reader = reader.reopen();
+ assertFalse("The index should change", oldReader == reader);
+ oldReader.close();
+ exposedSortTimes.add(testReopen(reader, query, plainSort));
+ }
+ reader.close();
+ long total = 0;
+ for (long t: exposedSortTimes) {
+ total += t;
+ }
+ System.out.println(
+ "\n" + RUNS + " exposed sortings with doc#-delta " + UPDATE_SIZE
+ + " between each took " + total
+ + " ms split in " + exposedSortTimes);
+ }
+
+ private long testReopen(
+ IndexReader reader, Query query, Sort plainSort) throws IOException {
+ final int docCount = reader.maxDoc();
+ final int MAX_HITS = 100;
+ Sort exposedSort = new Sort(new SortField(
+ "a",
+ new ExposedFieldComparatorSource(reader, new Locale("da"))));
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+ long plainTime = System.currentTimeMillis();
+ TopFieldDocs plainDocs = searcher.search(
+ query.createWeight(searcher), null, MAX_HITS, plainSort, true);
+ plainTime = System.currentTimeMillis() - plainTime;
+
+ long exposedTime = System.currentTimeMillis();
+ TopFieldDocs exposedDocs = searcher.search(
+ query.createWeight(searcher), null, MAX_HITS, exposedSort, true);
+ exposedTime = System.currentTimeMillis() - exposedTime;
+ System.out.println(
+ "Reopen sort test extracted "
+ + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits in "
+ + plainTime + " ms for standard Lucene collator sort and "
+ + exposedTime + " ms for exposed sort");
+// dumpDocs(reader, plainDocs, sortField);
+// dumpDocs(reader, exposedDocs, sortField);
+ assertEquals("The two search results should be equal",
+ plainDocs, exposedDocs);
+ return exposedTime;
+ }
+
+ public void testExposedSortWithNullValues() throws Exception {
+ testExposedSort(ExposedHelper.EVEN_NULL, DOCCOUNT, false);
+ }
+
+ public void testExposedSortWithRandom() throws Exception {
+ testExposedSort("a", DOCCOUNT, false);
+ }
+
+ public void testSpeedExposedSortWithRandom() throws Exception {
+// testExposedSort("a", 150000, true); Fails at GrowingMutable.get from compareBottom with java.lang.ArrayIndexOutOfBoundsException: 22674
+ testExposedSort("a", 20000, true);
+ }
+
+ private void testExposedSort(String sortField, int docCount, boolean feedback)
+ throws IOException, ParseException {
+ helper.createIndex(docCount, Arrays.asList("a", "b"), 20, 2);
+ testExposedSort(ExposedHelper.INDEX_LOCATION, new Locale("da"),
+ sortField, docCount, feedback);
+ }
+
+ public void testExternallyGeneratedIndex() throws Exception {
+ File external = new File("/home/te/projects/lucene4index");
+ if (!external.exists()) {
+ return;
+ }
+ testExposedSort(external, new Locale("da"), "sort_title", 10, true);
+ }
+
+ public void testExposedNaturalOrder() throws IOException, ParseException {
+ helper.createIndex(DOCCOUNT, Arrays.asList("a"), 20, 2);
+ testExposedSort(ExposedHelper.INDEX_LOCATION, null, "a", 10, true);
+ }
+
+ // locale can be null
+ private void testExposedSort(File index, Locale locale, String sortField,
+ int docCount, boolean feedback)
+ throws IOException, ParseException {
+
+ final int MAX_HITS = 50;
+ int retries = feedback ? 5 : 1;
+ IndexReader reader = IndexReader.open(FSDirectory.open(index), true);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser qp = new QueryParser(
+ Version.LUCENE_31, ExposedHelper.ALL, new MockAnalyzer());
+ Query q = qp.parse(ExposedHelper.ALL);
+ Sort aPlainSort = locale == null ?
+ new Sort(new SortField(sortField, SortField.STRING)) :
+ new Sort(new SortField(sortField, locale));
+ Sort aExposedSort = new Sort(new SortField(
+ sortField, new ExposedFieldComparatorSource(reader, locale)));
+
+ for (int run = 0 ; run < retries ; run++) {
+ long plainTime = System.currentTimeMillis();
+ TopFieldDocs plainDocs = searcher.search(
+ q.createWeight(searcher), null, MAX_HITS, aPlainSort, true);
+ plainTime = System.currentTimeMillis() - plainTime;
+
+ long exposedTime = System.currentTimeMillis();
+ TopFieldDocs exposedDocs = searcher.search(
+ q.createWeight(searcher), null, MAX_HITS, aExposedSort, true);
+ exposedTime = System.currentTimeMillis() - exposedTime;
+ System.out.println(
+ "Sorting on field " + sortField + " with "
+ + Math.min(MAX_HITS, docCount) + "/" + docCount + " hits took "
+ + plainTime + " ms for standard Lucene collator sort and "
+ + exposedTime + " ms for exposed sort");
+// dumpDocs(reader, plainDocs, sortField);
+// dumpDocs(reader, exposedDocs, sortField);
+ assertEquals("The two search results should be equal",
+ plainDocs, exposedDocs);
+ }
+ reader.close();
+ }
+
+ public void testRawSort() throws IOException {
+// final int[] SIZES = new int[]{10, 1000, 20000, 100000};
+ final int[] SIZES = new int[]{20000};
+ final int RUNS = 3;
+ Random random = new Random(87);
+ long createTime = System.currentTimeMillis();
+ for (int size: SIZES) {
+ List terms = new ArrayList(size);
+ for (int termPos = 0 ; termPos < size ; termPos++) {
+ String term = ExposedHelper.getRandomString(
+ random, ExposedHelper.CHARS, 1, 20) + termPos;
+ terms.add(new BytesRef(term));
+ }
+ System.out.println("Created " + size + " random BytesRefs in "
+ + (System.currentTimeMillis() - createTime) + "ms");
+
+ Comparator comparator = ExposedComparators.collatorToBytesRef(
+ Collator.getInstance(new Locale("da")));
+
+ for (int run = 0 ; run < RUNS ; run++) {
+ Collections.shuffle(terms);
+
+ long sortTime = System.currentTimeMillis();
+ Collections.sort(terms, comparator);
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println("Dumb sorted "
+ + ExposedUtil.time("bytesrefs", terms.size(), sortTime));
+
+ long keyTime = System.currentTimeMillis();
+ Collator plainCollator = Collator.getInstance(new Locale("da"));
+ List keys = new ArrayList(size);
+ for (BytesRef term: terms) {
+ keys.add(plainCollator.getCollationKey(term.utf8ToString()));
+ }
+ keyTime = System.currentTimeMillis() - keyTime;
+ System.out.println("Created "
+ + ExposedUtil.time("CollatorKeys", size, keyTime));
+ Collections.shuffle(keys);
+
+ long keySortTime = System.currentTimeMillis();
+ Collections.sort(keys);
+ keySortTime = System.currentTimeMillis() - keySortTime;
+ System.out.println("Collator sorted "
+ + ExposedUtil.time("CollatorKeys", size, keySortTime));
+ }
+ System.out.println("");
+ }
+ }
+
+ public void testPlainOrdinalAccess() throws IOException {
+ final int[] SIZES = new int[]{10, 1000, 50000};
+ final int RETRIES = 6;
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0];
+ TermsEnum terms = reader.fields().terms("a").iterator();
+ long num = reader.fields().terms("a").getUniqueTermCount();
+
+ Random random = new Random();
+ for (int i = 0 ; i < RETRIES ; i++) {
+ boolean doRandom = i % 2 == 0;
+ long firstHalf = System.currentTimeMillis();
+ for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) {
+ terms.seek(doRandom ? random.nextInt(ordinal+1) : ordinal);
+ BytesRef term = terms.term();
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ firstHalf = System.currentTimeMillis() - firstHalf;
+
+ long secondHalf = System.currentTimeMillis();
+ for (long ordinal = num / 2 ; ordinal < num ; ordinal++) {
+ terms.seek(doRandom ? random.nextInt((int) (ordinal+1)) : ordinal);
+ BytesRef term = terms.term();
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ secondHalf = System.currentTimeMillis() - secondHalf;
+
+ System.out.println("Seeked " + num + " ordinals "
+ + (doRandom ? "randomly" : "sequent.") + ". " +
+ "First half: " + firstHalf + " ms, second half: "
+ + secondHalf + " ms"
+ + (firstHalf + secondHalf == 0 ? "" : ". "
+ + num / (firstHalf + secondHalf) + " seeks/ms"));
+ }
+ reader.close();
+ }
+ }
+
+ public void testExposedOrdinalAccess() throws IOException {
+ final int[] SIZES = new int[]{10, 1000, 50000};
+ final int RETRIES = 6;
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents by exposed");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true).
+ getSequentialSubReaders()[0];
+ ExposedRequest.Field request = new ExposedRequest.Field("a", null, "foo");
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, request, true, true);
+ long num = provider.getMaxDoc();
+
+ Random random = new Random();
+ for (int i = 0 ; i < RETRIES ; i++) {
+ boolean doRandom = i % 2 == 0;
+ long firstHalf = System.currentTimeMillis();
+ for (int ordinal = 0 ; ordinal < num / 2 ; ordinal++) {
+ BytesRef term = provider.getTerm(
+ doRandom ? random.nextInt(ordinal+1) : ordinal);
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ firstHalf = System.currentTimeMillis() - firstHalf;
+
+ long secondHalf = System.currentTimeMillis();
+ for (long ordinal = num / 2 ; ordinal < num ; ordinal++) {
+ BytesRef term = provider.getTerm(
+ doRandom ? random.nextInt((int) (ordinal+1)) : ordinal);
+ assertNotNull(term); // To avoid optimization of terms.term()
+ }
+ secondHalf = System.currentTimeMillis() - secondHalf;
+
+ System.out.println("Requested " + (num-1) + " ordinals "
+ + (doRandom ? "randomly" : "sequent.") + ". " +
+ "First half: " + firstHalf + " ms, second half: "
+ + secondHalf + " ms"
+ + (firstHalf + secondHalf == 0 ? "" : ". "
+ + num / (firstHalf + secondHalf) + " seeks/ms"));
+ }
+ reader.close();
+ }
+ }
+ /*
+
+Measuring for 50000 documents
+Created 50000 document index with 4 fields with average term length 10 and total size 7MB in 2409ms
+org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@38a36b53
+org.apache.lucene.index.codecs.standard.StandardTermsDictReader$FieldReader@d694eca
+Chunk sorted 0-19999: 20000 ordinals in 1061 ms: ~= 18 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/255151, lookups=20000 (66 ms ~= 300 lookups/ms), readAheads=19999
+Chunk sorted 20000-25000: 5001 ordinals in 179 ms: ~= 27 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/54131, lookups=5001 (2 ms ~= 1722 lookups/ms), readAheads=5000
+Chunk merged 2 chunks in 234 ms: ~= 0 chunks/ms aka 25001 terms in 234 ms: ~= 106 terms/ms: CachedTermProvider cacheSize=20000, misses=15/49938, lookups=25804 (32 ms ~= 789 lookups/ms), readAheads=25797
+Chunk total sort for field a: 25001 terms in 1478 ms: ~= 16 terms/ms
+Chunk sorted 0-19999: 20000 ordinals in 523 ms: ~= 38 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/257147, lookups=20000 (12 ms ~= 1566 lookups/ms), readAheads=19999
+Chunk sorted 20000-24998: 4999 ordinals in 112 ms: ~= 44 ordinals/ms: CachedTermProvider cacheSize=20000, misses=1/53847, lookups=4999 (2 ms ~= 1752 lookups/ms), readAheads=4998
+Chunk merged 2 chunks in 116 ms: ~= 0 chunks/ms aka 24999 terms in 116 ms: ~= 215 terms/ms: CachedTermProvider cacheSize=20000, misses=21/49996, lookups=25217 (15 ms ~= 1674 lookups/ms), readAheads=25210
+Chunk total sort for field a: 24999 terms in 753 ms: ~= 33 terms/ms
+Group total iterator construction: 50000 ordinals in 2246 ms: ~= 22 ordinals/ms
+TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=411/25001, lookups=26406 (33 ms ~= 798 lookups/ms), readAheads=26181
+TermDocsIterator depleted: CachedTermProvider cacheSize=5000, misses=390/24999, lookups=26092 (32 ms ~= 799 lookups/ms), readAheads=25868
+Group ordinal iterator depletion from 2 providers: 50001 ordinals in 426 ms: ~= 117 ordinals/ms
+Got 50001 ordered ordinals in 2673ms: ~18 terms/ms
+
+ */
+ public void testTiming() throws IOException {
+// final int[] SIZES = new int[]{10, 1000, 20000, 50000};
+ final int[] SIZES = new int[]{20000};
+ for (int size: SIZES) {
+ System.out.println("\nMeasuring for " + size + " documents");
+ helper.close();
+ helper = new ExposedHelper();
+ helper.createIndex(size, Arrays.asList("a", "b"), 20, 2);
+ IndexReader reader = IndexReader.open(
+ FSDirectory.open(ExposedHelper.INDEX_LOCATION), true);
+ ExposedCache.getInstance().purgeAllCaches();
+
+ TermProvider provider = ExposedCache.getInstance().getProvider(
+ reader, "foo", Arrays.asList("a"),
+ ExposedComparators.collatorToBytesRef(Collator.getInstance(
+ new Locale("da"))), "bar");
+ assertEquals("The number of ordinal accessible terms should match",
+ size, provider.getOrdinalTermCount());
+
+ long sortTime = System.currentTimeMillis();
+ PackedInts.Reader orderedOrdinals = provider.getOrderedOrdinals();
+ sortTime = System.currentTimeMillis() - sortTime;
+ System.out.println("Got " + orderedOrdinals.size()
+ + " ordered ordinals in " + sortTime + "ms"
+ + (sortTime == 0 ? "" : ": ~" + (orderedOrdinals.size() / sortTime)
+ + " terms/ms"));
+ reader.close();
+/* Iterator iterator = provider.getIterator(false);
+ while (iterator.hasNext()) {
+ ExposedTuple tuple = iterator.next();
+ assertEquals("The provided term should match ordinal lookup term",
+ tuple.term.utf8ToString(),
+ provider.getTerm(tuple.ordinal).utf8ToString());
+ assertEquals("The provided term should match indirect lookup term",
+ tuple.term.utf8ToString(),
+ provider.getOrderedTerm(tuple.indirect).utf8ToString());
+ }
+
+ for (int docID = 0 ; docID < reader.maxDoc() ; docID++) {
+ String exposed = provider.getOrderedTerm(
+ provider.getDocToSingleIndirect().get(docID)).utf8ToString();
+ String direct = reader.document(docID).get("a");
+ assertEquals("Doc #" + docID + " should have the correct a-term",
+ direct, exposed);
+ }*/
+ }
+ }
+
+ public static void assertEquals(String message, TopFieldDocs expected,
+ TopFieldDocs actual) {
+ assertEquals(message + ". Expected length " + expected.scoreDocs.length
+ + " got " + actual.scoreDocs.length,
+ expected.scoreDocs.length, actual.scoreDocs.length);
+ for (int i = 0 ; i < actual.scoreDocs.length ; i++) {
+ ScoreDoc e = expected.scoreDocs[i];
+ ScoreDoc a = actual.scoreDocs[i];
+ assertEquals(message + ". The docID for hit#" + (i+1)
+ + "/" + expected.scoreDocs.length + " should be correct",
+ e.doc, a.doc);
+
+ String ef = ((FieldDoc)e).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)e).fields[0]).utf8ToString();
+ String af = ((FieldDoc)a).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)a).fields[0]).utf8ToString();
+ assertEquals(message + ". The sort value for hit#" + (i+1)
+ + "/" + expected.scoreDocs.length + " should be "
+ + ef + " but was " + af,
+ ef, af);
+ }
+ }
+
+ private void dumpDocs(IndexReader reader, TopFieldDocs docs, String field)
+ throws IOException {
+ final int MAX = 20;
+ int num = Math.min(docs.scoreDocs.length, MAX);
+ System.out.println("Dumping " + num +"/" + docs.scoreDocs.length + " hits");
+ int count = 0;
+ for (ScoreDoc doc: docs.scoreDocs) {
+ if (count++ > num) {
+ break;
+ }
+ String sortTerm = ((FieldDoc)doc).fields[0] == null ? "null"
+ : ((BytesRef)((FieldDoc)doc).fields[0]).utf8ToString();
+ System.out.println(
+ doc.doc + " ID=" + reader.document(doc.doc).get(ExposedHelper.ID)
+ + ", field " + field + "=" + reader.document(doc.doc).get(field)
+ + ", sort term=" + sortTerm);
+ }
+ }
+
+
+}
Index: lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java Tue Jul 13 12:41:02 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/ExposedTimSort.java Tue Jul 13 12:41:02 CEST 2010
@@ -0,0 +1,920 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.exposed;
+
+import java.util.Arrays;
+
+/**
+ * A stable, adaptive, iterative mergesort that requires far fewer than
+ * n lg(n) comparisons when running on partially sorted arrays, while
+ * offering performance comparable to a traditional mergesort when run
+ * on random arrays. Like all proper mergesorts, this sort is stable and
+ * runs O(n log n) time (worst case). In the worst case, this sort requires
+ * temporary storage space for n/2 object references; in the best case,
+ * it requires only a small constant amount of space.
+ *
+ * This implementation was adapted from Tim Peters's list sort for
+ * Python, which is described in detail here:
+ *
+ * http://svn.python.org/projects/python/trunk/Objects/listsort.txt
+ *
+ * Tim's C code may be found here:
+ *
+ * http://svn.python.org/projects/python/trunk/Objects/listobject.c
+ *
+ * The underlying techniques are described in this paper (and may have
+ * even earlier origins):
+ *
+ * "Optimistic Sorting and Information Theoretic Complexity"
+ * Peter McIlroy
+ * SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms),
+ * pp 467-474, Austin, Texas, 25-27 January 1993.
+ *
+ * While the API to this class consists solely of static methods, it is
+ * (privately) instantiable; a TimSort instance holds the state of an ongoing
+ * sort, assuming the input array is large enough to warrant the full-blown
+ * TimSort. Small arrays are sorted in place, using a binary insertion sort.
+ *
+ * @author Josh Bloch
+ *
+ * Modified 2010-03-22 to handle int only (specialization to save memory)
+ * by Toke Eskildsen (te@statsbiblioteket.dk).
+ */
+public class ExposedTimSort {
+ /**
+ * This is the minimum sized sequence that will be merged. Shorter
+ * sequences will be lengthened by calling binarySort. If the entire
+ * array is less than this length, no merges will be performed.
+ *
+ * This constant should be a power of two. It was 64 in Tim Peter's C
+ * implementation, but 32 was empirically determined to work better in
+ * this implementation. In the unlikely event that you set this constant
+ * to be a number that's not a power of two, you'll need to change the
+ * {@link #minRunLength} computation.
+ *
+ * If you decrease this constant, you must change the stackLen
+ * computation in the TimSort constructor, or you risk an
+ * ArrayOutOfBounds exception. See listsort.txt for a discussion
+ * of the minimum stack length required as a function of the length
+ * of the array being sorted and the minimum merge sequence length.
+ */
+ private static final int MIN_MERGE = 32;
+
+ /**
+ * The array being sorted.
+ */
+ private final int[] a;
+
+ /**
+ * The comparator for this sort.
+ */
+ private final ExposedComparators.OrdinalComparator c;
+
+ /**
+ * When we get into galloping mode, we stay there until both runs win less
+ * often than MIN_GALLOP consecutive times.
+ */
+ private static final int MIN_GALLOP = 7;
+
+ /**
+ * This controls when we get *into* galloping mode. It is initialized
+ * to MIN_GALLOP. The mergeLo and mergeHi methods nudge it higher for
+ * random data, and lower for highly structured data.
+ */
+ private int minGallop = MIN_GALLOP;
+
+ /**
+ * Maximum initial size of tmp array, which is used for merging. The array
+ * can grow to accommodate demand.
+ *
+ * Unlike Tim's original C version, we do not allocate this much storage
+ * when sorting smaller arrays. This change was required for performance.
+ */
+ private static final int INITIAL_TMP_STORAGE_LENGTH = 256;
+
+ /**
+ * Temp storage for merges.
+ */
+ private int[] tmp; // Actual runtime type will be Object[], regardless of T
+
+ /**
+ * A stack of pending runs yet to be merged. Run i starts at
+ * address base[i] and extends for len[i] elements. It's always
+ * true (so long as the indices are in bounds) that:
+ *
+ * runBase[i] + runLen[i] == runBase[i + 1]
+ *
+ * so we could cut the storage for this, but it's a minor amount,
+ * and keeping all the info explicit simplifies the code.
+ */
+ private int stackSize = 0; // Number of pending runs on stack
+ private final int[] runBase;
+ private final int[] runLen;
+
+ /**
+ * Creates a TimSort instance to maintain the state of an ongoing sort.
+ *
+ * @param a the array to be sorted
+ * @param c the comparator to determine the order of the sort
+ */
+ private ExposedTimSort(int[] a, ExposedComparators.OrdinalComparator c) {
+ this.a = a;
+ this.c = c;
+
+ // Allocate temp storage (which may be increased later if necessary)
+ int len = a.length;
+ tmp = new int[len < 2 * INITIAL_TMP_STORAGE_LENGTH ?
+ len >>> 1 : INITIAL_TMP_STORAGE_LENGTH];
+
+ /*
+ * Allocate runs-to-be-merged stack (which cannot be expanded). The
+ * stack length requirements are described in listsort.txt. The C
+ * version always uses the same stack length (85), but this was
+ * measured to be too expensive when sorting "mid-sized" arrays (e.g.,
+ * 100 elements) in Java. Therefore, we use smaller (but sufficiently
+ * large) stack lengths for smaller arrays. The "magic numbers" in the
+ * computation below must be changed if MIN_MERGE is decreased. See
+ * the MIN_MERGE declaration above for more information.
+ */
+ int stackLen = (len < 120 ? 5 :
+ len < 1542 ? 10 :
+ len < 119151 ? 19 : 40);
+ runBase = new int[stackLen];
+ runLen = new int[stackLen];
+ }
+
+ /*
+ * The next two methods (which are package private and static) constitute
+ * the entire API of this class. Each of these methods obeys the contract
+ * of the public method with the same signature in java.util.Arrays.
+ */
+
+ public static void sort(int[] a, ExposedComparators.OrdinalComparator c) {
+ sort(a, 0, a.length, c);
+ }
+
+ public static void sort(int[] a, int lo, int hi, ExposedComparators.OrdinalComparator c) {
+ if (c == null) {
+ Arrays.sort(a, lo, hi);
+ return;
+ }
+
+ rangeCheck(a.length, lo, hi);
+ int nRemaining = hi - lo;
+ if (nRemaining < 2)
+ return; // Arrays of size 0 and 1 are always sorted
+
+ // If array is small, do a "mini-TimSort" with no merges
+ if (nRemaining < MIN_MERGE) {
+ int initRunLen = countRunAndMakeAscending(a, lo, hi, c);
+ binarySort(a, lo, hi, lo + initRunLen, c);
+ return;
+ }
+
+ /**
+ * March over the array once, left to right, finding natural runs,
+ * extending short natural runs to minRun elements, and merging runs
+ * to maintain stack invariant.
+ */
+ ExposedTimSort ts = new ExposedTimSort(a, c);
+ int minRun = minRunLength(nRemaining);
+ do {
+ // Identify next run
+ int runLen = countRunAndMakeAscending(a, lo, hi, c);
+
+ // If run is short, extend to min(minRun, nRemaining)
+ if (runLen < minRun) {
+ int force = nRemaining <= minRun ? nRemaining : minRun;
+ binarySort(a, lo, lo + force, lo + runLen, c);
+ runLen = force;
+ }
+
+ // Push run onto pending-run stack, and maybe merge
+ ts.pushRun(lo, runLen);
+ ts.mergeCollapse();
+
+ // Advance to find next run
+ lo += runLen;
+ nRemaining -= runLen;
+ } while (nRemaining != 0);
+
+ // Merge all remaining runs to complete sort
+ assert lo == hi;
+ ts.mergeForceCollapse();
+ assert ts.stackSize == 1;
+ }
+
+ /**
+ * Sorts the specified portion of the specified array using a binary
+ * insertion sort. This is the best method for sorting small numbers
+ * of elements. It requires O(n log n) compares, but O(n^2) data
+ * movement (worst case).
+ *
+ * If the initial part of the specified range is already sorted,
+ * this method can take advantage of it: the method assumes that the
+ * elements from index {@code lo}, inclusive, to {@code start},
+ * exclusive are already sorted.
+ *
+ * @param a the array in which a range is to be sorted
+ * @param lo the index of the first element in the range to be sorted
+ * @param hi the index after the last element in the range to be sorted
+ * @param start the index of the first element in the range that is
+ * not already known to be sorted (@code lo <= start <= hi}
+ * @param c comparator to used for the sort
+ */
+ @SuppressWarnings("fallthrough")
+ private static void binarySort(int[] a, int lo, int hi, int start,
+ ExposedComparators.OrdinalComparator c) {
+ assert lo <= start && start <= hi;
+ if (start == lo)
+ start++;
+ for ( ; start < hi; start++) {
+ int pivot = a[start];
+
+ // Set left (and right) to the index where a[start] (pivot) belongs
+ int left = lo;
+ int right = start;
+ assert left <= right;
+ /*
+ * Invariants:
+ * pivot >= all in [lo, left).
+ * pivot < all in [right, start).
+ */
+ while (left < right) {
+ int mid = (left + right) >>> 1;
+ if (c.compare(pivot, a[mid]) < 0)
+ right = mid;
+ else
+ left = mid + 1;
+ }
+ assert left == right;
+
+ /*
+ * The invariants still hold: pivot >= all in [lo, left) and
+ * pivot < all in [left, start), so pivot belongs at left. Note
+ * that if there are elements equal to pivot, left points to the
+ * first slot after them -- that's why this sort is stable.
+ * Slide elements over to make room to make room for pivot.
+ */
+ int n = start - left; // The number of elements to move
+ // Switch is just an optimization for arraycopy in default case
+ switch(n) {
+ case 2: a[left + 2] = a[left + 1];
+ case 1: a[left + 1] = a[left];
+ break;
+ default: System.arraycopy(a, left, a, left + 1, n);
+ }
+ a[left] = pivot;
+ }
+ }
+
+ /**
+ * Returns the length of the run beginning at the specified position in
+ * the specified array and reverses the run if it is descending (ensuring
+ * that the run will always be ascending when the method returns).
+ *
+ * A run is the longest ascending sequence with:
+ *
+ * a[lo] <= a[lo + 1] <= a[lo + 2] <= ...
+ *
+ * or the longest descending sequence with:
+ *
+ * a[lo] > a[lo + 1] > a[lo + 2] > ...
+ *
+ * For its intended use in a stable mergesort, the strictness of the
+ * definition of "descending" is needed so that the call can safely
+ * reverse a descending sequence without violating stability.
+ *
+ * @param a the array in which a run is to be counted and possibly reversed
+ * @param lo index of the first element in the run
+ * @param hi index after the last element that may be contained in the run.
+ It is required that @code{lo < hi}.
+ * @param c the comparator to used for the sort
+ * @return the length of the run beginning at the specified position in
+ * the specified array
+ */
+ private static int countRunAndMakeAscending(int[] a, int lo, int hi,
+ ExposedComparators.OrdinalComparator c) {
+ assert lo < hi;
+ int runHi = lo + 1;
+ if (runHi == hi)
+ return 1;
+
+ // Find end of run, and reverse range if descending
+ if (c.compare(a[runHi++], a[lo]) < 0) { // Descending
+ while(runHi < hi && c.compare(a[runHi], a[runHi - 1]) < 0)
+ runHi++;
+ reverseRange(a, lo, runHi);
+ } else { // Ascending
+ while (runHi < hi && c.compare(a[runHi], a[runHi - 1]) >= 0)
+ runHi++;
+ }
+
+ return runHi - lo;
+ }
+
+ /**
+ * Reverse the specified range of the specified array.
+ *
+ * @param a the array in which a range is to be reversed
+ * @param lo the index of the first element in the range to be reversed
+ * @param hi the index after the last element in the range to be reversed
+ */
+ private static void reverseRange(int[] a, int lo, int hi) {
+ hi--;
+ while (lo < hi) {
+ int t = a[lo];
+ a[lo++] = a[hi];
+ a[hi--] = t;
+ }
+ }
+
+ /**
+ * Returns the minimum acceptable run length for an array of the specified
+ * length. Natural runs shorter than this will be extended with
+ * {@link #binarySort}.
+ *
+ * Roughly speaking, the computation is:
+ *
+ * If n < MIN_MERGE, return n (it's too small to bother with fancy stuff).
+ * Else if n is an exact power of 2, return MIN_MERGE/2.
+ * Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k
+ * is close to, but strictly less than, an exact power of 2.
+ *
+ * For the rationale, see listsort.txt.
+ *
+ * @param n the length of the array to be sorted
+ * @return the length of the minimum run to be merged
+ */
+ private static int minRunLength(int n) {
+ assert n >= 0;
+ int r = 0; // Becomes 1 if any 1 bits are shifted off
+ while (n >= MIN_MERGE) {
+ r |= (n & 1);
+ n >>= 1;
+ }
+ return n + r;
+ }
+
+ /**
+ * Pushes the specified run onto the pending-run stack.
+ *
+ * @param runBase index of the first element in the run
+ * @param runLen the number of elements in the run
+ */
+ private void pushRun(int runBase, int runLen) {
+ this.runBase[stackSize] = runBase;
+ this.runLen[stackSize] = runLen;
+ stackSize++;
+ }
+
+ /**
+ * Examines the stack of runs waiting to be merged and merges adjacent runs
+ * until the stack invariants are reestablished:
+ *
+ * 1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1]
+ * 2. runLen[i - 2] > runLen[i - 1]
+ *
+ * This method is called each time a new run is pushed onto the stack,
+ * so the invariants are guaranteed to hold for i < stackSize upon
+ * entry to the method.
+ */
+ private void mergeCollapse() {
+ while (stackSize > 1) {
+ int n = stackSize - 2;
+ if (n > 0 && runLen[n-1] <= runLen[n] + runLen[n+1]) {
+ if (runLen[n - 1] < runLen[n + 1])
+ n--;
+ mergeAt(n);
+ } else if (runLen[n] <= runLen[n + 1]) {
+ mergeAt(n);
+ } else {
+ break; // Invariant is established
+ }
+ }
+ }
+
+ /**
+ * Merges all runs on the stack until only one remains. This method is
+ * called once, to complete the sort.
+ */
+ private void mergeForceCollapse() {
+ while (stackSize > 1) {
+ int n = stackSize - 2;
+ if (n > 0 && runLen[n - 1] < runLen[n + 1])
+ n--;
+ mergeAt(n);
+ }
+ }
+
+ /**
+ * Merges the two runs at stack indices i and i+1. Run i must be
+ * the penultimate or antepenultimate run on the stack. In other words,
+ * i must be equal to stackSize-2 or stackSize-3.
+ *
+ * @param i stack index of the first of the two runs to merge
+ */
+ private void mergeAt(int i) {
+ assert stackSize >= 2;
+ assert i >= 0;
+ assert i == stackSize - 2 || i == stackSize - 3;
+
+ int base1 = runBase[i];
+ int len1 = runLen[i];
+ int base2 = runBase[i + 1];
+ int len2 = runLen[i + 1];
+ assert len1 > 0 && len2 > 0;
+ assert base1 + len1 == base2;
+
+ /*
+ * Record the length of the combined runs; if i is the 3rd-last
+ * run now, also slide over the last run (which isn't involved
+ * in this merge). The current run (i+1) goes away in any case.
+ */
+ runLen[i] = len1 + len2;
+ if (i == stackSize - 3) {
+ runBase[i + 1] = runBase[i + 2];
+ runLen[i + 1] = runLen[i + 2];
+ }
+ stackSize--;
+
+ /*
+ * Find where the first element of run2 goes in run1. Prior elements
+ * in run1 can be ignored (because they're already in place).
+ */
+ int k = gallopRight(a[base2], a, base1, len1, 0, c);
+ assert k >= 0;
+ base1 += k;
+ len1 -= k;
+ if (len1 == 0)
+ return;
+
+ /*
+ * Find where the last element of run1 goes in run2. Subsequent elements
+ * in run2 can be ignored (because they're already in place).
+ */
+ len2 = gallopLeft(a[base1 + len1 - 1], a, base2, len2, len2 - 1, c);
+ assert len2 >= 0;
+ if (len2 == 0)
+ return;
+
+ // Merge remaining runs, using tmp array with min(len1, len2) elements
+ if (len1 <= len2)
+ mergeLo(base1, len1, base2, len2);
+ else
+ mergeHi(base1, len1, base2, len2);
+ }
+
+ /**
+ * Locates the position at which to insert the specified key into the
+ * specified sorted range; if the range contains an element equal to key,
+ * returns the index of the leftmost equal element.
+ *
+ * @param key the key whose insertion point to search for
+ * @param a the array in which to search
+ * @param base the index of the first element in the range
+ * @param len the length of the range; must be > 0
+ * @param hint the index at which to begin the search, 0 <= hint < n.
+ * The closer hint is to the result, the faster this method will run.
+ * @param c the comparator used to order the range, and to search
+ * @return the int k, 0 <= k <= n such that a[b + k - 1] < key <= a[b + k],
+ * pretending that a[b - 1] is minus infinity and a[b + n] is infinity.
+ * In other words, key belongs at index b + k; or in other words,
+ * the first k elements of a should precede key, and the last n - k
+ * should follow it.
+ */
+ private static int gallopLeft(int key, int[] a, int base, int len, int hint,
+ ExposedComparators.OrdinalComparator c) {
+ assert len > 0 && hint >= 0 && hint < len;
+ int lastOfs = 0;
+ int ofs = 1;
+ if (c.compare(key, a[base + hint]) > 0) {
+ // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs]
+ int maxOfs = len - hint;
+ while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) > 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to base
+ lastOfs += hint;
+ ofs += hint;
+ } else { // key <= a[base + hint]
+ // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs]
+ final int maxOfs = hint + 1;
+ while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) <= 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to base
+ int tmp = lastOfs;
+ lastOfs = hint - ofs;
+ ofs = hint - tmp;
+ }
+ assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
+
+ /*
+ * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere
+ * to the right of lastOfs but no farther right than ofs. Do a binary
+ * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs].
+ */
+ lastOfs++;
+ while (lastOfs < ofs) {
+ int m = lastOfs + ((ofs - lastOfs) >>> 1);
+
+ if (c.compare(key, a[base + m]) > 0)
+ lastOfs = m + 1; // a[base + m] < key
+ else
+ ofs = m; // key <= a[base + m]
+ }
+ assert lastOfs == ofs; // so a[base + ofs - 1] < key <= a[base + ofs]
+ return ofs;
+ }
+
+ /**
+ * Like gallopLeft, except that if the range contains an element equal to
+ * key, gallopRight returns the index after the rightmost equal element.
+ *
+ * @param key the key whose insertion point to search for
+ * @param a the array in which to search
+ * @param base the index of the first element in the range
+ * @param len the length of the range; must be > 0
+ * @param hint the index at which to begin the search, 0 <= hint < n.
+ * The closer hint is to the result, the faster this method will run.
+ * @param c the comparator used to order the range, and to search
+ * @return the int k, 0 <= k <= n such that a[b + k - 1] <= key < a[b + k]
+ */
+ private static int gallopRight(int key, int[] a, int base, int len,
+ int hint, ExposedComparators.OrdinalComparator c) {
+ assert len > 0 && hint >= 0 && hint < len;
+
+ int ofs = 1;
+ int lastOfs = 0;
+ if (c.compare(key, a[base + hint]) < 0) {
+ // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs]
+ int maxOfs = hint + 1;
+ while (ofs < maxOfs && c.compare(key, a[base + hint - ofs]) < 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to b
+ int tmp = lastOfs;
+ lastOfs = hint - ofs;
+ ofs = hint - tmp;
+ } else { // a[b + hint] <= key
+ // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs]
+ int maxOfs = len - hint;
+ while (ofs < maxOfs && c.compare(key, a[base + hint + ofs]) >= 0) {
+ lastOfs = ofs;
+ ofs = (ofs << 1) + 1;
+ if (ofs <= 0) // int overflow
+ ofs = maxOfs;
+ }
+ if (ofs > maxOfs)
+ ofs = maxOfs;
+
+ // Make offsets relative to b
+ lastOfs += hint;
+ ofs += hint;
+ }
+ assert -1 <= lastOfs && lastOfs < ofs && ofs <= len;
+
+ /*
+ * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to
+ * the right of lastOfs but no farther right than ofs. Do a binary
+ * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs].
+ */
+ lastOfs++;
+ while (lastOfs < ofs) {
+ int m = lastOfs + ((ofs - lastOfs) >>> 1);
+
+ if (c.compare(key, a[base + m]) < 0)
+ ofs = m; // key < a[b + m]
+ else
+ lastOfs = m + 1; // a[b + m] <= key
+ }
+ assert lastOfs == ofs; // so a[b + ofs - 1] <= key < a[b + ofs]
+ return ofs;
+ }
+
+ /**
+ * Merges two adjacent runs in place, in a stable fashion. The first
+ * element of the first run must be greater than the first element of the
+ * second run (a[base1] > a[base2]), and the last element of the first run
+ * (a[base1 + len1-1]) must be greater than all elements of the second run.
+ *
+ * For performance, this method should be called only when len1 <= len2;
+ * its twin, mergeHi should be called if len1 >= len2. (Either method
+ * may be called if len1 == len2.)
+ *
+ * @param base1 index of first element in first run to be merged
+ * @param len1 length of first run to be merged (must be > 0)
+ * @param base2 index of first element in second run to be merged
+ * (must be aBase + aLen)
+ * @param len2 length of second run to be merged (must be > 0)
+ */
+ private void mergeLo(int base1, int len1, int base2, int len2) {
+ assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
+
+ // Copy first run into temp array
+ int[] a = this.a; // For performance
+ int[] tmp = ensureCapacity(len1);
+ System.arraycopy(a, base1, tmp, 0, len1);
+
+ int cursor1 = 0; // Indexes into tmp array
+ int cursor2 = base2; // Indexes int a
+ int dest = base1; // Indexes int a
+
+ // Move first element of second run and deal with degenerate cases
+ a[dest++] = a[cursor2++];
+ if (--len2 == 0) {
+ System.arraycopy(tmp, cursor1, a, dest, len1);
+ return;
+ }
+ if (len1 == 1) {
+ System.arraycopy(a, cursor2, a, dest, len2);
+ a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge
+ return;
+ }
+
+ ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance
+ int minGallop = this.minGallop; // " " " " "
+ outer:
+ while (true) {
+ int count1 = 0; // Number of times in a row that first run won
+ int count2 = 0; // Number of times in a row that second run won
+
+ /*
+ * Do the straightforward thing until (if ever) one run starts
+ * winning consistently.
+ */
+ do {
+ assert len1 > 1 && len2 > 0;
+ if (c.compare(a[cursor2], tmp[cursor1]) < 0) {
+ a[dest++] = a[cursor2++];
+ count2++;
+ count1 = 0;
+ if (--len2 == 0)
+ break outer;
+ } else {
+ a[dest++] = tmp[cursor1++];
+ count1++;
+ count2 = 0;
+ if (--len1 == 1)
+ break outer;
+ }
+ } while ((count1 | count2) < minGallop);
+
+ /*
+ * One run is winning so consistently that galloping may be a
+ * huge win. So try that, and continue galloping until (if ever)
+ * neither run appears to be winning consistently anymore.
+ */
+ do {
+ assert len1 > 1 && len2 > 0;
+ count1 = gallopRight(a[cursor2], tmp, cursor1, len1, 0, c);
+ if (count1 != 0) {
+ System.arraycopy(tmp, cursor1, a, dest, count1);
+ dest += count1;
+ cursor1 += count1;
+ len1 -= count1;
+ if (len1 <= 1) // len1 == 1 || len1 == 0
+ break outer;
+ }
+ a[dest++] = a[cursor2++];
+ if (--len2 == 0)
+ break outer;
+
+ count2 = gallopLeft(tmp[cursor1], a, cursor2, len2, 0, c);
+ if (count2 != 0) {
+ System.arraycopy(a, cursor2, a, dest, count2);
+ dest += count2;
+ cursor2 += count2;
+ len2 -= count2;
+ if (len2 == 0)
+ break outer;
+ }
+ a[dest++] = tmp[cursor1++];
+ if (--len1 == 1)
+ break outer;
+ minGallop--;
+ } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
+ if (minGallop < 0)
+ minGallop = 0;
+ minGallop += 2; // Penalize for leaving gallop mode
+ } // End of "outer" loop
+ this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field
+
+ if (len1 == 1) {
+ assert len2 > 0;
+ System.arraycopy(a, cursor2, a, dest, len2);
+ a[dest + len2] = tmp[cursor1]; // Last elt of run 1 to end of merge
+ } else if (len1 == 0) {
+ throw new IllegalArgumentException(
+ "Comparison method violates its general contract!");
+ } else {
+ assert len2 == 0;
+ assert len1 > 1;
+ System.arraycopy(tmp, cursor1, a, dest, len1);
+ }
+ }
+
+ /**
+ * Like mergeLo, except that this method should be called only if
+ * len1 >= len2; mergeLo should be called if len1 <= len2. (Either method
+ * may be called if len1 == len2.)
+ *
+ * @param base1 index of first element in first run to be merged
+ * @param len1 length of first run to be merged (must be > 0)
+ * @param base2 index of first element in second run to be merged
+ * (must be aBase + aLen)
+ * @param len2 length of second run to be merged (must be > 0)
+ */
+ private void mergeHi(int base1, int len1, int base2, int len2) {
+ assert len1 > 0 && len2 > 0 && base1 + len1 == base2;
+
+ // Copy second run into temp array
+ int[] a = this.a; // For performance
+ int[] tmp = ensureCapacity(len2);
+ System.arraycopy(a, base2, tmp, 0, len2);
+
+ int cursor1 = base1 + len1 - 1; // Indexes into a
+ int cursor2 = len2 - 1; // Indexes into tmp array
+ int dest = base2 + len2 - 1; // Indexes into a
+
+ // Move last element of first run and deal with degenerate cases
+ a[dest--] = a[cursor1--];
+ if (--len1 == 0) {
+ System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2);
+ return;
+ }
+ if (len2 == 1) {
+ dest -= len1;
+ cursor1 -= len1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, len1);
+ a[dest] = tmp[cursor2];
+ return;
+ }
+
+ ExposedComparators.OrdinalComparator c = this.c; // Use local variable for performance
+ int minGallop = this.minGallop; // " " " " "
+ outer:
+ while (true) {
+ int count1 = 0; // Number of times in a row that first run won
+ int count2 = 0; // Number of times in a row that second run won
+
+ /*
+ * Do the straightforward thing until (if ever) one run
+ * appears to win consistently.
+ */
+ do {
+ assert len1 > 0 && len2 > 1;
+ if (c.compare(tmp[cursor2], a[cursor1]) < 0) {
+ a[dest--] = a[cursor1--];
+ count1++;
+ count2 = 0;
+ if (--len1 == 0)
+ break outer;
+ } else {
+ a[dest--] = tmp[cursor2--];
+ count2++;
+ count1 = 0;
+ if (--len2 == 1)
+ break outer;
+ }
+ } while ((count1 | count2) < minGallop);
+
+ /*
+ * One run is winning so consistently that galloping may be a
+ * huge win. So try that, and continue galloping until (if ever)
+ * neither run appears to be winning consistently anymore.
+ */
+ do {
+ assert len1 > 0 && len2 > 1;
+ count1 = len1 - gallopRight(tmp[cursor2], a, base1, len1, len1 - 1, c);
+ if (count1 != 0) {
+ dest -= count1;
+ cursor1 -= count1;
+ len1 -= count1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, count1);
+ if (len1 == 0)
+ break outer;
+ }
+ a[dest--] = tmp[cursor2--];
+ if (--len2 == 1)
+ break outer;
+
+ count2 = len2 - gallopLeft(a[cursor1], tmp, 0, len2, len2 - 1, c);
+ if (count2 != 0) {
+ dest -= count2;
+ cursor2 -= count2;
+ len2 -= count2;
+ System.arraycopy(tmp, cursor2 + 1, a, dest + 1, count2);
+ if (len2 <= 1) // len2 == 1 || len2 == 0
+ break outer;
+ }
+ a[dest--] = a[cursor1--];
+ if (--len1 == 0)
+ break outer;
+ minGallop--;
+ } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP);
+ if (minGallop < 0)
+ minGallop = 0;
+ minGallop += 2; // Penalize for leaving gallop mode
+ } // End of "outer" loop
+ this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field
+
+ if (len2 == 1) {
+ assert len1 > 0;
+ dest -= len1;
+ cursor1 -= len1;
+ System.arraycopy(a, cursor1 + 1, a, dest + 1, len1);
+ a[dest] = tmp[cursor2]; // Move first elt of run2 to front of merge
+ } else if (len2 == 0) {
+ throw new IllegalArgumentException(
+ "Comparison method violates its general contract!");
+ } else {
+ assert len1 == 0;
+ assert len2 > 0;
+ System.arraycopy(tmp, 0, a, dest - (len2 - 1), len2);
+ }
+ }
+
+ /**
+ * Ensures that the external array tmp has at least the specified
+ * number of elements, increasing its size if necessary. The size
+ * increases exponentially to ensure amortized linear time complexity.
+ *
+ * @param minCapacity the minimum required capacity of the tmp array
+ * @return tmp, whether or not it grew
+ */
+ private int[] ensureCapacity(int minCapacity) {
+ if (tmp.length < minCapacity) {
+ // Compute smallest power of 2 > minCapacity
+ int newSize = minCapacity;
+ newSize |= newSize >> 1;
+ newSize |= newSize >> 2;
+ newSize |= newSize >> 4;
+ newSize |= newSize >> 8;
+ newSize |= newSize >> 16;
+ newSize++;
+
+ if (newSize < 0) // Not bloody likely!
+ newSize = minCapacity;
+ else
+ newSize = Math.min(newSize, a.length >>> 1);
+
+ tmp = new int[newSize];
+ }
+ return tmp;
+ }
+
+ /**
+ * Checks that fromIndex and toIndex are in range, and throws an
+ * appropriate exception if they aren't.
+ *
+ * @param arrayLen the length of the array
+ * @param fromIndex the index of the first element of the range
+ * @param toIndex the index after the last element of the range
+ * @throws IllegalArgumentException if fromIndex > toIndex
+ * @throws ArrayIndexOutOfBoundsException if fromIndex < 0
+ * or toIndex > arrayLen
+ */
+ private static void rangeCheck(int arrayLen, int fromIndex, int toIndex) {
+ if (fromIndex > toIndex)
+ throw new IllegalArgumentException("fromIndex(" + fromIndex +
+ ") > toIndex(" + toIndex+")");
+ if (fromIndex < 0)
+ throw new ArrayIndexOutOfBoundsException(fromIndex);
+ if (toIndex > arrayLen)
+ throw new ArrayIndexOutOfBoundsException(toIndex);
+ }
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java Thu Sep 02 14:05:13 CEST 2010
+++ lucene/src/java/org/apache/lucene/search/exposed/TermDocIterator.java Thu Sep 02 14:05:13 CEST 2010
@@ -0,0 +1,126 @@
+package org.apache.lucene.search.exposed;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Single source and single field Iterator that provides terms and potentially
+ * document IDs.
+ *
+ * It is recommended to use a {@link CachedTermProvider} as source or otherwise
+ * ensure that ordered iteration is fast if no docIDs are to be returned..
+ * If docIDs are to be returned, caching at the source level will not help.
+ *
+ * Important: The ExposedTuple is reused between calls to {@link #next()} unless
+ * {@link #setReuseTuple(boolean)} is called with false.
+ */
+public class TermDocIterator implements Iterator {
+ private final TermProvider source;
+ private final PackedInts.Reader order;
+ private final boolean collectDocIDs;
+
+ private DocsEnum docsEnum = null;
+
+ private int position = 0;
+ private boolean reuseTuple = true;
+
+ private boolean pending = false; // true == Terms or docIDs are ready in tuple
+ private final ExposedTuple tuple;
+
+ public TermDocIterator(TermProvider source, boolean collectDocIDs)
+ throws IOException {
+ this.source = source;
+ this.collectDocIDs = collectDocIDs;
+ order = source.getOrderedOrdinals();
+ String field = source.getField(0);
+
+ tuple = new ExposedTuple(field, null, 0, 0, -1);
+ }
+
+ // Ensure that we can deliver an id
+ public boolean hasNext() {
+ while (true) {
+ if (pending) {
+ return true;
+ }
+ if (position >= order.size()) {
+ if (source instanceof CachedTermProvider) {
+ System.out.println("TermDocsIterator depleted: "
+ + ((CachedTermProvider)source).getStats());
+ }
+ return false;
+ }
+ try {
+ seekToNextTerm();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next docID", e);
+ }
+ }
+ }
+
+ public ExposedTuple next() {
+ if (!hasNext()) {
+ throw new IllegalStateException("The iterator is depleted");
+ }
+ if (!collectDocIDs) {
+ pending = false;
+ } else {
+ tuple.docID = docsEnum.docID();
+ try {
+ if (docsEnum.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
+ pending = false;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "IOException while seeking to next document", e);
+ }
+ }
+ return reuseTuple ? tuple : new ExposedTuple(tuple);
+ }
+
+ private void seekToNextTerm() throws IOException {
+ while (true) {
+ pending = false;
+ if (position >= order.size()) {
+ return;
+ }
+ tuple.indirect = position;
+ tuple.ordinal = order.get(position);
+ try {
+ tuple.term = source.getTerm(tuple.ordinal);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to resolve the term at order="
+ + order + ", ordinal=" + order.get(position), e);
+ }
+ position++;
+
+ if (!collectDocIDs) {
+ pending = true;
+ return;
+ }
+
+ docsEnum = source.getDocsEnum(tuple.ordinal, docsEnum);
+ int doc;
+ if ((doc = docsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS
+ && !(source.getReader().hasDeletions()
+ && source.getReader().getDeletedDocs().get(doc))) {
+ pending = true;
+ return; // We break at first genuine docID
+ }
+ }
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("Not a valid operation");
+ }
+
+ public void setReuseTuple(boolean reuseTuple) {
+ this.reuseTuple = reuseTuple;
+ }
+}
Index: lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java
===================================================================
--- lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java Wed May 26 15:23:06 CEST 2010
+++ lucene/src/test/org/apache/lucene/util/packed/TestGrowingMutable.java Wed May 26 15:23:06 CEST 2010
@@ -0,0 +1,47 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestGrowingMutable extends LuceneTestCase {
+
+ public void testMiscGrowth() throws Exception {
+ GrowingMutable g = new GrowingMutable();
+
+ g.set(100, 1000);
+ assertEquals("The value at index 100 should be correct", 1000, g.get(100));
+ assertEquals("The size should be correct", 1, g.size());
+
+ g.set(98, 5000);
+ assertEquals("The value at index 98 should still be correct",
+ 5000, g.get(98));
+ assertEquals("The value at index 100 should still be correct",
+ 1000, g.get(100));
+ assertEquals("The size should be extended", 3, g.size());
+
+ g.set(-123, -12345);
+ assertEquals("The value at index 98 should still be correct",
+ 5000, g.get(98));
+ assertEquals("The value at index 100 should still be correct",
+ 1000, g.get(100));
+ assertEquals("The value at -123 should be as defined",
+ -12345, g.get(-123));
+ }
+
+}