());
+
public LuceneTestCase() {
super();
}
@@ -64,7 +85,18 @@
@Override
protected void setUp() throws Exception {
super.setUp();
+
+ savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread t, Throwable e) {
+ uncaughtExceptions.add(new UncaughtExceptionEntry(t, e));
+ if (savedUncaughtExceptionHandler != null)
+ savedUncaughtExceptionHandler.uncaughtException(t, e);
+ }
+ });
+
ConcurrentMergeScheduler.setTestMode();
+ savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount();
}
/**
@@ -87,6 +119,7 @@
@Override
protected void tearDown() throws Exception {
+ BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
try {
// this isn't as useful as calling directly from the scope where the
// index readers are used, because they could be gc'ed just before
@@ -104,6 +137,16 @@
purgeFieldCache(FieldCache.DEFAULT);
}
+ Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
+ if (!uncaughtExceptions.isEmpty()) {
+ System.err.println("The following exceptions were thrown by threads:");
+ for (UncaughtExceptionEntry entry : uncaughtExceptions) {
+ System.err.println("*** Thread: " + entry.thread.getName() + " ***");
+ entry.exception.printStackTrace(System.err);
+ }
+ fail("Some threads throwed uncaught exceptions!");
+ }
+
super.tearDown();
}
Index: src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java
===================================================================
--- src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestIndexableBinaryStringTools.java (working copy)
@@ -164,14 +164,14 @@
int encodedLen1 = IndexableBinaryStringTools.getEncodedLength(
originalArray1, 0, numBytes1);
if (encodedLen1 > encoded1.length)
- encoded1 = new char[ArrayUtil.getNextSize(encodedLen1)];
+ encoded1 = new char[ArrayUtil.oversize(encodedLen1, RamUsageEstimator.NUM_BYTES_CHAR)];
IndexableBinaryStringTools.encode(originalArray1, 0, numBytes1, encoded1,
0, encodedLen1);
int encodedLen2 = IndexableBinaryStringTools.getEncodedLength(original2,
0, numBytes2);
if (encodedLen2 > encoded2.length)
- encoded2 = new char[ArrayUtil.getNextSize(encodedLen2)];
+ encoded2 = new char[ArrayUtil.oversize(encodedLen2, RamUsageEstimator.NUM_BYTES_CHAR)];
IndexableBinaryStringTools.encode(original2, 0, numBytes2, encoded2, 0,
encodedLen2);
@@ -308,7 +308,7 @@
int encodedLen = IndexableBinaryStringTools.getEncodedLength(binary, 0,
numBytes);
if (encoded.length < encodedLen)
- encoded = new char[ArrayUtil.getNextSize(encodedLen)];
+ encoded = new char[ArrayUtil.oversize(encodedLen, RamUsageEstimator.NUM_BYTES_CHAR)];
IndexableBinaryStringTools.encode(binary, 0, numBytes, encoded, 0,
encodedLen);
Index: src/test/org/apache/lucene/util/ArrayUtilTest.java
===================================================================
--- src/test/org/apache/lucene/util/ArrayUtilTest.java (revision 916567)
+++ src/test/org/apache/lucene/util/ArrayUtilTest.java (working copy)
@@ -1,74 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.util.LuceneTestCase;
-
-
-/**
- *
- *
- **/
-public class ArrayUtilTest extends LuceneTestCase {
-
- public void testParseInt() throws Exception {
- int test;
- try {
- test = ArrayUtil.parseInt("".toCharArray());
- assertTrue(false);
- } catch (NumberFormatException e) {
- //expected
- }
- try {
- test = ArrayUtil.parseInt("foo".toCharArray());
- assertTrue(false);
- } catch (NumberFormatException e) {
- //expected
- }
- try {
- test = ArrayUtil.parseInt(String.valueOf(Long.MAX_VALUE).toCharArray());
- assertTrue(false);
- } catch (NumberFormatException e) {
- //expected
- }
- try {
- test = ArrayUtil.parseInt("0.34".toCharArray());
- assertTrue(false);
- } catch (NumberFormatException e) {
- //expected
- }
-
- try {
- test = ArrayUtil.parseInt("1".toCharArray());
- assertTrue(test + " does not equal: " + 1, test == 1);
- test = ArrayUtil.parseInt("-10000".toCharArray());
- assertTrue(test + " does not equal: " + -10000, test == -10000);
- test = ArrayUtil.parseInt("1923".toCharArray());
- assertTrue(test + " does not equal: " + 1923, test == 1923);
- test = ArrayUtil.parseInt("-1".toCharArray());
- assertTrue(test + " does not equal: " + -1, test == -1);
- test = ArrayUtil.parseInt("foo 1923 bar".toCharArray(), 4, 4);
- assertTrue(test + " does not equal: " + 1923, test == 1923);
- } catch (NumberFormatException e) {
- e.printStackTrace();
- assertTrue(false);
- }
-
- }
-
-}
Index: src/test/org/apache/lucene/util/TestOpenBitSet.java
===================================================================
--- src/test/org/apache/lucene/util/TestOpenBitSet.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestOpenBitSet.java (working copy)
@@ -22,9 +22,6 @@
import org.apache.lucene.search.DocIdSetIterator;
-/**
- * @version $Id$
- */
public class TestOpenBitSet extends LuceneTestCase {
Random rand;
@@ -233,7 +230,14 @@
}
}
-
+ public void testHashCodeEquals() {
+ OpenBitSet bs1 = new OpenBitSet(200);
+ OpenBitSet bs2 = new OpenBitSet(64);
+ bs1.set(3);
+ bs2.set(3);
+ assertEquals(bs1, bs2);
+ assertEquals(bs1.hashCode(), bs2.hashCode());
+ }
}
Property changes on: src\test\org\apache\lucene\util\TestAttributeSource.java
___________________________________________________________________
Added: svn:mergeinfo
Merged /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:r909334
Index: src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
===================================================================
--- src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (working copy)
@@ -45,9 +45,9 @@
RAMDirectory dirA = new RAMDirectory();
RAMDirectory dirB = new RAMDirectory();
- IndexWriter wA = new IndexWriter(dirA, new WhitespaceAnalyzer(), true,
+ IndexWriter wA = new IndexWriter(dirA, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
- IndexWriter wB = new IndexWriter(dirB, new WhitespaceAnalyzer(), true,
+ IndexWriter wB = new IndexWriter(dirB, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), true,
IndexWriter.MaxFieldLength.LIMITED);
long theLong = Long.MAX_VALUE;
Index: src/test/org/apache/lucene/util/English.java
===================================================================
--- src/test/org/apache/lucene/util/English.java (revision 916567)
+++ src/test/org/apache/lucene/util/English.java (working copy)
@@ -1,186 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-public class English {
-
- public static String longToEnglish(long i) {
- StringBuilder result = new StringBuilder();
- longToEnglish(i, result);
- return result.toString();
- }
-
- public static void longToEnglish(long i, StringBuilder result) {
- if (i == 0) {
- result.append("zero");
- return;
- }
- if (i < 0) {
- result.append("minus ");
- i = -i;
- }
- if (i >= 1000000000000000000l) { // quadrillion
- longToEnglish(i / 1000000000000000000l, result);
- result.append("quintillion, ");
- i = i % 1000000000000000000l;
- }
- if (i >= 1000000000000000l) { // quadrillion
- longToEnglish(i / 1000000000000000l, result);
- result.append("quadrillion, ");
- i = i % 1000000000000000l;
- }
- if (i >= 1000000000000l) { // trillions
- longToEnglish(i / 1000000000000l, result);
- result.append("trillion, ");
- i = i % 1000000000000l;
- }
- if (i >= 1000000000) { // billions
- longToEnglish(i / 1000000000, result);
- result.append("billion, ");
- i = i % 1000000000;
- }
- if (i >= 1000000) { // millions
- longToEnglish(i / 1000000, result);
- result.append("million, ");
- i = i % 1000000;
- }
- if (i >= 1000) { // thousands
- longToEnglish(i / 1000, result);
- result.append("thousand, ");
- i = i % 1000;
- }
- if (i >= 100) { // hundreds
- longToEnglish(i / 100, result);
- result.append("hundred ");
- i = i % 100;
- }
- //we know we are smaller here so we can cast
- if (i >= 20) {
- switch (((int) i) / 10) {
- case 9:
- result.append("ninety");
- break;
- case 8:
- result.append("eighty");
- break;
- case 7:
- result.append("seventy");
- break;
- case 6:
- result.append("sixty");
- break;
- case 5:
- result.append("fifty");
- break;
- case 4:
- result.append("forty");
- break;
- case 3:
- result.append("thirty");
- break;
- case 2:
- result.append("twenty");
- break;
- }
- i = i % 10;
- if (i == 0)
- result.append(" ");
- else
- result.append("-");
- }
- switch ((int) i) {
- case 19:
- result.append("nineteen ");
- break;
- case 18:
- result.append("eighteen ");
- break;
- case 17:
- result.append("seventeen ");
- break;
- case 16:
- result.append("sixteen ");
- break;
- case 15:
- result.append("fifteen ");
- break;
- case 14:
- result.append("fourteen ");
- break;
- case 13:
- result.append("thirteen ");
- break;
- case 12:
- result.append("twelve ");
- break;
- case 11:
- result.append("eleven ");
- break;
- case 10:
- result.append("ten ");
- break;
- case 9:
- result.append("nine ");
- break;
- case 8:
- result.append("eight ");
- break;
- case 7:
- result.append("seven ");
- break;
- case 6:
- result.append("six ");
- break;
- case 5:
- result.append("five ");
- break;
- case 4:
- result.append("four ");
- break;
- case 3:
- result.append("three ");
- break;
- case 2:
- result.append("two ");
- break;
- case 1:
- result.append("one ");
- break;
- case 0:
- result.append("");
- break;
- }
- }
-
-
- public static String intToEnglish(int i) {
- StringBuilder result = new StringBuilder();
- longToEnglish(i, result);
- return result.toString();
- }
-
- public static void intToEnglish(int i, StringBuilder result) {
- longToEnglish(i, result);
- }
-
- public static void main(String[] args) {
- System.out.println(longToEnglish(Long.parseLong(args[0])));
- }
-
-}
Index: src/test/org/apache/lucene/util/TestRamUsageEstimator.java
===================================================================
--- src/test/org/apache/lucene/util/TestRamUsageEstimator.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestRamUsageEstimator.java (working copy)
@@ -25,20 +25,20 @@
String string = new String("test str");
RamUsageEstimator rue = new RamUsageEstimator();
long size = rue.estimateRamUsage(string);
- System.out.println("size:" + size);
+ //System.out.println("size:" + size);
string = new String("test strin");
size = rue.estimateRamUsage(string);
- System.out.println("size:" + size);
+ //System.out.println("size:" + size);
Holder holder = new Holder();
holder.holder = new Holder("string2", 5000L);
size = rue.estimateRamUsage(holder);
- System.out.println("size:" + size);
+ //System.out.println("size:" + size);
String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")};
size = rue.estimateRamUsage(strings);
- System.out.println("size:" + size);
+ //System.out.println("size:" + size);
}
private static final class Holder {
Index: src/test/org/apache/lucene/util/TestSortedVIntList.java
===================================================================
--- src/test/org/apache/lucene/util/TestSortedVIntList.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestSortedVIntList.java (working copy)
@@ -20,8 +20,6 @@
import java.io.IOException;
import java.util.BitSet;
-import org.apache.lucene.util.LuceneTestCase;
-
import junit.framework.TestSuite;
import junit.textui.TestRunner;
@@ -193,4 +191,11 @@
public void test12() {
tstIllegalArgExc(new int[] {0,1,1,2,3,5,8,0});
}
+ public void test13Allocation() throws Exception {
+ int [] a = new int[2000]; // SortedVIntList initial byte size is 128
+ for (int i = 0; i < a.length; i++) {
+ a[i] = (107 + i) * i;
+ }
+ tstIterator(new SortedVIntList(a), a);
+ }
}
Index: src/test/org/apache/lucene/util/TestSmallFloat.java
===================================================================
--- src/test/org/apache/lucene/util/TestSmallFloat.java (revision 916567)
+++ src/test/org/apache/lucene/util/TestSmallFloat.java (working copy)
@@ -18,9 +18,6 @@
import java.util.Random;
-/**
- * @version $Id$
- */
public class TestSmallFloat extends LuceneTestCase {
// original lucene byteToFloat
Index: src/test/org/apache/lucene/document/TestBinaryDocument.java
===================================================================
--- src/test/org/apache/lucene/document/TestBinaryDocument.java (revision 916567)
+++ src/test/org/apache/lucene/document/TestBinaryDocument.java (working copy)
@@ -59,7 +59,7 @@
/** add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.addDocument(doc);
writer.close();
@@ -97,7 +97,7 @@
/** add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.addDocument(doc);
writer.close();
Property changes on: src\test\org\apache\lucene\document\TestNumberTools.java
___________________________________________________________________
Modified: svn:mergeinfo
Merged /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:r909334
Merged /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:r889614-889621,889623-889666,889668-889865,899002-916543
Property changes on: src\test\org\apache\lucene\document\TestDateTools.java
___________________________________________________________________
Modified: svn:mergeinfo
Merged /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:r889614-889621,889623-889666,889668-889865,899002-916543
Merged /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:r909334
Index: src/test/org/apache/lucene/document/TestDocument.java
===================================================================
--- src/test/org/apache/lucene/document/TestDocument.java (revision 916567)
+++ src/test/org/apache/lucene/document/TestDocument.java (working copy)
@@ -154,7 +154,7 @@
public void testGetValuesForIndexedDocument() throws Exception
{
RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.addDocument(makeDocumentWithFields());
writer.close();
@@ -225,7 +225,7 @@
doc.add(new Field("keyword", "test", Field.Store.YES, Field.Index.NOT_ANALYZED));
RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(TEST_VERSION_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.addDocument(doc);
field.setValue("id2");
writer.addDocument(doc);
Index: src/java/org/apache/lucene/analysis/SimpleAnalyzer.java
===================================================================
--- src/java/org/apache/lucene/analysis/SimpleAnalyzer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/SimpleAnalyzer.java (working copy)
@@ -19,14 +19,42 @@
import java.io.Reader;
+import org.apache.lucene.util.Version;
+
/** An {@link Analyzer} that filters {@link LetterTokenizer}
- * with {@link LowerCaseFilter} */
-
+ * with {@link LowerCaseFilter}
+ *
+ * You must specify the required {@link Version} compatibility
+ * when creating {@link CharTokenizer}:
+ *
+ * - As of 3.1, {@link LowerCaseTokenizer} uses an int based API to normalize and
+ * detect token codepoints. See {@link CharTokenizer#isTokenChar(int)} and
+ * {@link CharTokenizer#normalize(int)} for details.
+ *
+ *
+ **/
public final class SimpleAnalyzer extends ReusableAnalyzerBase {
+ private final Version matchVersion;
+
+ /**
+ * Creates a new {@link SimpleAnalyzer}
+ * @param matchVersion Lucene version to match See {@link above}
+ */
+ public SimpleAnalyzer(Version matchVersion) {
+ this.matchVersion = matchVersion;
+ }
+
+ /**
+ * Creates a new {@link SimpleAnalyzer}
+ * @deprecated use {@link #SimpleAnalyzer(Version)} instead
+ */
+ @Deprecated public SimpleAnalyzer() {
+ this(Version.LUCENE_30);
+ }
@Override
protected TokenStreamComponents createComponents(final String fieldName,
final Reader reader) {
- return new TokenStreamComponents(new LowerCaseTokenizer(reader));
+ return new TokenStreamComponents(new LowerCaseTokenizer(matchVersion, reader));
}
}
Index: src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (working copy)
@@ -1,10 +1,5 @@
package org.apache.lucene.analysis;
-import java.io.IOException;
-
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.ArrayUtil;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -22,6 +17,12 @@
* limitations under the License.
*/
+import java.io.IOException;
+
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
+
/**
* This class converts alphabetic, numeric, and symbolic Unicode characters
* which are not in the first 127 ASCII characters (the "Basic Latin" Unicode
@@ -101,7 +102,7 @@
// Worst-case length required:
final int maxSizeNeeded = 4 * length;
if (output.length < maxSizeNeeded) {
- output = new char[ArrayUtil.getNextSize(maxSizeNeeded)];
+ output = new char[ArrayUtil.oversize(maxSizeNeeded, RamUsageEstimator.NUM_BYTES_CHAR)];
}
outputPos = 0;
Index: src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java
===================================================================
--- src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/WhitespaceAnalyzer.java (working copy)
@@ -19,13 +19,44 @@
import java.io.Reader;
-/** An Analyzer that uses {@link WhitespaceTokenizer}. */
+import org.apache.lucene.util.Version;
+/**
+ * An Analyzer that uses {@link WhitespaceTokenizer}.
+ *
+ * You must specify the required {@link Version} compatibility
+ * when creating {@link CharTokenizer}:
+ *
+ * - As of 3.1, {@link WhitespaceTokenizer} uses an int based API to normalize and
+ * detect token codepoints. See {@link CharTokenizer#isTokenChar(int)} and
+ * {@link CharTokenizer#normalize(int)} for details.
+ *
+ *
+ **/
public final class WhitespaceAnalyzer extends ReusableAnalyzerBase {
-
+
+ private final Version matchVersion;
+
+ /**
+ * Creates a new {@link WhitespaceAnalyzer}
+ * @param matchVersion Lucene version to match See {@link above}
+ */
+ public WhitespaceAnalyzer(Version matchVersion) {
+ this.matchVersion = matchVersion;
+ }
+
+ /**
+ * Creates a new {@link WhitespaceAnalyzer}
+ * @deprecated use {@link #WhitespaceAnalyzer(Version)} instead
+ */
+ @Deprecated
+ public WhitespaceAnalyzer() {
+ this(Version.LUCENE_30);
+ }
+
@Override
protected TokenStreamComponents createComponents(final String fieldName,
final Reader reader) {
- return new TokenStreamComponents(new WhitespaceTokenizer(reader));
+ return new TokenStreamComponents(new WhitespaceTokenizer(matchVersion, reader));
}
}
Index: src/java/org/apache/lucene/analysis/NumericTokenStream.java
===================================================================
--- src/java/org/apache/lucene/analysis/NumericTokenStream.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/NumericTokenStream.java (working copy)
@@ -82,8 +82,7 @@
* href="../search/NumericRangeQuery.html#precisionStepDesc">precisionStep
* parameter as well as how numeric fields work under the hood.
*
- * NOTE: This API is experimental and
- * might change in incompatible ways in the next release.
+ * @lucene.experimental
*
* @since 2.9
*/
Index: src/java/org/apache/lucene/analysis/CharFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/CharFilter.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/CharFilter.java (working copy)
@@ -24,9 +24,6 @@
* They can be used as {@link java.io.Reader} with additional offset
* correction. {@link Tokenizer}s will automatically use {@link #correctOffset}
* if a CharFilter/CharStream subclass is used.
- *
- * @version $Id$
- *
*/
public abstract class CharFilter extends CharStream {
Index: src/java/org/apache/lucene/analysis/WordlistLoader.java
===================================================================
--- src/java/org/apache/lucene/analysis/WordlistLoader.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/WordlistLoader.java (working copy)
@@ -191,6 +191,66 @@
return result;
}
+ /**
+ * Loads a text file in Snowball format associated with a given class (See
+ * {@link Class#getResourceAsStream(String)}) and adds all words as entries to
+ * a {@link Set}. The words need to be in lower-case if you make use of an
+ * Analyzer which uses LowerCaseFilter (like StandardAnalyzer).
+ *
+ * @param aClass a class that is associated with the given stopwordResource
+ * @param stopwordResource name of the resource file associated with the given
+ * class
+ * @return a {@link Set} with the file's words
+ * @see #getSnowballWordSet(Reader)
+ */
+ public static Set getSnowballWordSet(Class aClass,
+ String stopwordResource) throws IOException {
+ final Reader reader = new BufferedReader(new InputStreamReader(aClass
+ .getResourceAsStream(stopwordResource), "UTF-8"));
+ try {
+ return getSnowballWordSet(reader);
+ } finally {
+ reader.close();
+ }
+ }
+
+ /**
+ * Reads stopwords from a stopword list in Snowball format.
+ *
+ * The snowball format is the following:
+ *
+ * - Lines may contain multiple words separated by whitespace.
+ *
- The comment character is the vertical line (|).
+ *
- Lines may contain trailing comments.
+ *
+ *
+ *
+ * @param reader Reader containing a Snowball stopword list
+ * @return A Set with the reader's words
+ */
+ public static Set getSnowballWordSet(Reader reader)
+ throws IOException {
+ final Set result = new HashSet();
+ BufferedReader br = null;
+ try {
+ if (reader instanceof BufferedReader) {
+ br = (BufferedReader) reader;
+ } else {
+ br = new BufferedReader(reader);
+ }
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ int comment = line.indexOf('|');
+ if (comment >= 0) line = line.substring(0, comment);
+ String words[] = line.split("\\s+");
+ for (int i = 0; i < words.length; i++)
+ if (words[i].length() > 0) result.add(words[i]);
+ }
+ } finally {
+ if (br != null) br.close();
+ }
+ return result;
+ }
/**
Index: src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java
===================================================================
--- src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java (working copy)
@@ -17,6 +17,8 @@
* limitations under the License.
*/
+import org.apache.lucene.document.Fieldable;
+
import java.io.Reader;
import java.io.IOException;
import java.util.Map;
@@ -118,6 +120,15 @@
analyzer = defaultAnalyzer;
return analyzer.getPositionIncrementGap(fieldName);
}
+
+ /** Return the offsetGap from the analyzer assigned to field */
+ @Override
+ public int getOffsetGap(Fieldable field) {
+ Analyzer analyzer = analyzerMap.get(field.name());
+ if (analyzer == null)
+ analyzer = defaultAnalyzer;
+ return analyzer.getOffsetGap(field);
+ }
@Override
public String toString() {
Index: src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java (working copy)
@@ -23,10 +23,10 @@
/**
* This attribute can be used to pass different flags down the {@link Tokenizer} chain,
* eg from one TokenFilter to another one.
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
*/
public interface FlagsAttribute extends Attribute {
/**
- * EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
*
*
* Get the bitset for any bits that have been set. This is completely distinct from {@link TypeAttribute#type()}, although they do share similar purposes.
Index: src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java (working copy)
@@ -24,12 +24,12 @@
/**
* This attribute can be used to pass different flags down the tokenizer chain,
* eg from one TokenFilter to another one.
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
*/
public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute, Cloneable, Serializable {
private int flags = 0;
/**
- * EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
*
*
* Get the bitset for any bits that have been set. This is completely distinct from {@link TypeAttribute#type()}, although they do share similar purposes.
Index: src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java (working copy)
@@ -21,6 +21,7 @@
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* The term text of a Token.
@@ -106,12 +107,12 @@
public char[] resizeTermBuffer(int newSize) {
if (termBuffer == null) {
// The buffer is always at least MIN_BUFFER_SIZE
- termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
} else {
if(termBuffer.length < newSize){
// Not big enough; create a new array with slight
// over allocation and preserve content
- final char[] newCharBuffer = new char[ArrayUtil.getNextSize(newSize)];
+ final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
termBuffer = newCharBuffer;
}
@@ -127,19 +128,19 @@
private void growTermBuffer(int newSize) {
if (termBuffer == null) {
// The buffer is always at least MIN_BUFFER_SIZE
- termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
} else {
if(termBuffer.length < newSize){
// Not big enough; create a new array with slight
// over allocation:
- termBuffer = new char[ArrayUtil.getNextSize(newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
}
}
}
private void initTermBuffer() {
if (termBuffer == null) {
- termBuffer = new char[ArrayUtil.getNextSize(MIN_BUFFER_SIZE)];
+ termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
termLength = 0;
}
}
Index: src/java/org/apache/lucene/analysis/CharTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/CharTokenizer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/CharTokenizer.java (working copy)
@@ -23,59 +23,310 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.CharacterUtils;
+import org.apache.lucene.util.Version;
+import org.apache.lucene.util.VirtualMethod;
+import org.apache.lucene.util.CharacterUtils.CharacterBuffer;
-/** An abstract base class for simple, character-oriented tokenizers.*/
+/**
+ * An abstract base class for simple, character-oriented tokenizers.
+ *
+ * You must specify the required {@link Version} compatibility
+ * when creating {@link CharTokenizer}:
+ *
+ * - As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
+ * detect token codepoints. See {@link #isTokenChar(int)} and
+ * {@link #normalize(int)} for details.
+ *
+ *
+ * A new {@link CharTokenizer} API has been introduced with Lucene 3.1. This API
+ * moved from UTF-16 code units to UTF-32 codepoints to eventually add support
+ * for supplementary characters. The old char based API has been
+ * deprecated and should be replaced with the int based methods
+ * {@link #isTokenChar(int)} and {@link #normalize(int)}.
+ *
+ *
+ * As of Lucene 3.1 each {@link CharTokenizer} - constructor expects a
+ * {@link Version} argument. Based on the given {@link Version} either the new
+ * API or a backwards compatibility layer is used at runtime. For
+ * {@link Version} < 3.1 the backwards compatibility layer ensures correct
+ * behavior even for indexes build with previous versions of Lucene. If a
+ * {@link Version} >= 3.1 is used {@link CharTokenizer} requires the new API to
+ * be implemented by the instantiated class. Yet, the old char based API
+ * is not required anymore even if backwards compatibility must be preserved.
+ * {@link CharTokenizer} subclasses implementing the new API are fully backwards
+ * compatible if instantiated with {@link Version} < 3.1.
+ *
+ *
+ * Note: If you use a subclass of {@link CharTokenizer} with {@link Version} >=
+ * 3.1 on an index build with a version < 3.1, created tokens might not be
+ * compatible with the terms in your index.
+ *
+ **/
public abstract class CharTokenizer extends Tokenizer {
- public CharTokenizer(Reader input) {
+
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param input
+ * the input to split up into tokens
+ */
+ public CharTokenizer(Version matchVersion, Reader input) {
super(input);
+ charUtils = CharacterUtils.getInstance(matchVersion);
offsetAtt = addAttribute(OffsetAttribute.class);
termAtt = addAttribute(TermAttribute.class);
+ useOldAPI = useOldAPI(matchVersion);
+ ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
+
}
-
- public CharTokenizer(AttributeSource source, Reader input) {
+
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param source
+ * the attribute source to use for this {@link Tokenizer}
+ * @param input
+ * the input to split up into tokens
+ */
+ public CharTokenizer(Version matchVersion, AttributeSource source,
+ Reader input) {
super(source, input);
+ charUtils = CharacterUtils.getInstance(matchVersion);
offsetAtt = addAttribute(OffsetAttribute.class);
termAtt = addAttribute(TermAttribute.class);
+ useOldAPI = useOldAPI(matchVersion);
+ ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
}
-
- public CharTokenizer(AttributeFactory factory, Reader input) {
+
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param factory
+ * the attribute factory to use for this {@link Tokenizer}
+ * @param input
+ * the input to split up into tokens
+ */
+ public CharTokenizer(Version matchVersion, AttributeFactory factory,
+ Reader input) {
super(factory, input);
+ charUtils = CharacterUtils.getInstance(matchVersion);
offsetAtt = addAttribute(OffsetAttribute.class);
termAtt = addAttribute(TermAttribute.class);
+ useOldAPI = useOldAPI(matchVersion);
+ ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
}
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ * @param input the input to split up into tokens
+ * @deprecated use {@link #CharTokenizer(Version, Reader)} instead. This will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
+ public CharTokenizer(Reader input) {
+ this(Version.LUCENE_30, input);
+ }
+
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ * @param input the input to split up into tokens
+ * @param source the attribute source to use for this {@link Tokenizer}
+ * @deprecated use {@link #CharTokenizer(Version, AttributeSource, Reader)} instead. This will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
+ public CharTokenizer(AttributeSource source, Reader input) {
+ this(Version.LUCENE_30, source, input);
+ }
+
+ /**
+ * Creates a new {@link CharTokenizer} instance
+ * @param input the input to split up into tokens
+ * @param factory the attribute factory to use for this {@link Tokenizer}
+ * @deprecated use {@link #CharTokenizer(Version, AttributeSource.AttributeFactory, Reader)} instead. This will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
+ public CharTokenizer(AttributeFactory factory, Reader input) {
+ this(Version.LUCENE_30, factory, input);
+ }
+
private int offset = 0, bufferIndex = 0, dataLen = 0;
private static final int MAX_WORD_LEN = 255;
private static final int IO_BUFFER_SIZE = 4096;
- private final char[] ioBuffer = new char[IO_BUFFER_SIZE];
- private TermAttribute termAtt;
- private OffsetAttribute offsetAtt;
+ private final TermAttribute termAtt;
+ private final OffsetAttribute offsetAtt;
+
+ private final CharacterUtils charUtils;
+ private final CharacterBuffer ioBuffer;
+
+ /**
+ * @deprecated this will be removed in lucene 4.0
+ */
+ @Deprecated
+ private final boolean useOldAPI;
+
+ /**
+ * @deprecated this will be removed in lucene 4.0
+ */
+ @Deprecated
+ private static final VirtualMethod isTokenCharMethod =
+ new VirtualMethod(CharTokenizer.class, "isTokenChar", char.class);
+
+ /**
+ * @deprecated this will be removed in lucene 4.0
+ */
+ @Deprecated
+ private static final VirtualMethod normalizeMethod =
+ new VirtualMethod(CharTokenizer.class, "normalize", char.class);
- /** Returns true iff a character should be included in a token. This
+ /**
+ * Returns true iff a UTF-16 code unit should be included in a token. This
* tokenizer generates as tokens adjacent sequences of characters which
- * satisfy this predicate. Characters for which this is false are used to
- * define token boundaries and are not included in tokens. */
- protected abstract boolean isTokenChar(char c);
+ * satisfy this predicate. Characters for which this is false are
+ * used to define token boundaries and are not included in tokens.
+ *
+ * Note: This method cannot handle supplementary characters. To support all Unicode characters, including
+ * supplementary characters, use the {@link #isTokenChar(int)} method.
+ *
+ *
+ * @deprecated use {@link #isTokenChar(int)} instead. This method will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
+ protected boolean isTokenChar(char c) {
+ return isTokenChar((int)c);
+ }
- /** Called on each token character to normalize it before it is added to the
- * token. The default implementation does nothing. Subclasses may use this
- * to, e.g., lowercase tokens. */
+ /**
+ * Called on each token UTF-16 code unit to normalize it before it is added to the
+ * token. The default implementation does nothing. Subclasses may use this to,
+ * e.g., lowercase tokens.
+ *
+ * Note: This method cannot handle supplementary characters. To support all Unicode characters, including
+ * supplementary characters, use the {@link #normalize(int)} method.
+ *
+ *
+ * @deprecated use {@link #normalize(int)} instead. This method will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
protected char normalize(char c) {
+ return (char) normalize((int) c);
+ }
+
+ /**
+ * Returns true iff a codepoint should be included in a token. This tokenizer
+ * generates as tokens adjacent sequences of codepoints which satisfy this
+ * predicate. Codepoints for which this is false are used to define token
+ * boundaries and are not included in tokens.
+ *
+ * As of Lucene 3.1 the char based API ({@link #isTokenChar(char)} and
+ * {@link #normalize(char)}) has been depreciated in favor of a Unicode 4.0
+ * compatible int based API to support codepoints instead of UTF-16 code
+ * units. Subclasses of {@link CharTokenizer} must not override the char based
+ * methods if a {@link Version} >= 3.1 is passed to the constructor.
+ *
+ *
+ * NOTE: This method will be marked abstract in Lucene 4.0.
+ *
+ */
+ protected boolean isTokenChar(int c) {
+ throw new UnsupportedOperationException("since LUCENE_3_1 subclasses of CharTokenizer must implement isTokenChar(int)");
+ }
+
+ /**
+ * Called on each token character to normalize it before it is added to the
+ * token. The default implementation does nothing. Subclasses may use this to,
+ * e.g., lowercase tokens.
+ *
+ * As of Lucene 3.1 the char based API ({@link #isTokenChar(char)} and
+ * {@link #normalize(char)}) has been depreciated in favor of a Unicode 4.0
+ * compatible int based API to support codepoints instead of UTF-16 code
+ * units. Subclasses of {@link CharTokenizer} must not override the char based
+ * methods if a {@link Version} >= 3.1 is passed to the constructor.
+ *
+ *
+ * NOTE: This method will be marked abstract in Lucene 4.0.
+ *
+ */
+ protected int normalize(int c) {
return c;
}
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
+ if(useOldAPI) // TODO remove this in LUCENE 4.0
+ return incrementTokenOld();
int length = 0;
int start = bufferIndex;
char[] buffer = termAtt.termBuffer();
while (true) {
+ if (bufferIndex >= dataLen) {
+ offset += dataLen;
+ if(!charUtils.fill(ioBuffer, input)) { // read supplementary char aware with CharacterUtils
+ dataLen = 0; // so next offset += dataLen won't decrement offset
+ if (length > 0)
+ break;
+ else
+ return false;
+ }
+ dataLen = ioBuffer.getLength();
+ bufferIndex = 0;
+ }
+ // use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone
+ final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex);
+ bufferIndex += Character.charCount(c);
+ if (isTokenChar(c)) { // if it's a token char
+ if (length == 0) // start of token
+ start = offset + bufferIndex - 1;
+ else if (length >= buffer.length-1) // check if a supplementary could run out of bounds
+ buffer = termAtt.resizeTermBuffer(2+length); // make sure a supplementary fits in the buffer
+ length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
+ if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test
+ break;
+ } else if (length > 0) // at non-Letter w/ chars
+ break; // return 'em
+ }
+
+ termAtt.setTermLength(length);
+ offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
+ return true;
+
+ }
+
+ /**
+ * The <= 3.0 version of incrementToken. This is a backwards compat implementation used
+ * if a version <= 3.0 is provided to the ctor.
+ * @deprecated remove in 4.0
+ */
+ @Deprecated
+ private boolean incrementTokenOld() throws IOException {
+ int length = 0;
+ int start = bufferIndex;
+ char[] buffer = termAtt.termBuffer();
+ final char[] oldIoBuffer = ioBuffer.getBuffer();
+ while (true) {
+
if (bufferIndex >= dataLen) {
offset += dataLen;
- dataLen = input.read(ioBuffer);
+ dataLen = input.read(oldIoBuffer);
if (dataLen == -1) {
dataLen = 0; // so next offset += dataLen won't decrement offset
if (length > 0)
@@ -86,7 +337,7 @@
bufferIndex = 0;
}
- final char c = ioBuffer[bufferIndex++];
+ final char c = oldIoBuffer[bufferIndex++];
if (isTokenChar(c)) { // if it's a token char
@@ -107,12 +358,14 @@
termAtt.setTermLength(length);
offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
return true;
- }
+ }
+
+
@Override
public final void end() {
// set final offset
- int finalOffset = correctOffset(offset);
+ final int finalOffset = correctOffset(offset);
offsetAtt.setOffset(finalOffset, finalOffset);
}
@@ -122,5 +375,19 @@
bufferIndex = 0;
offset = 0;
dataLen = 0;
+ ioBuffer.reset(); // make sure to reset the IO buffer!!
}
-}
+
+ /**
+ * @deprecated this will be removed in lucene 4.0
+ */
+ @Deprecated
+ private boolean useOldAPI(Version matchVersion) {
+ final Class clazz = this.getClass();
+ if (matchVersion.onOrAfter(Version.LUCENE_31)
+ && (isTokenCharMethod.isOverriddenAsOf(clazz) || normalizeMethod
+ .isOverriddenAsOf(clazz))) throw new IllegalArgumentException(
+ "For matchVersion >= LUCENE_31, CharTokenizer subclasses must not override isTokenChar(char) or normalize(char).");
+ return !matchVersion.onOrAfter(Version.LUCENE_31);
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/analysis/PorterStemmer.java
===================================================================
--- src/java/org/apache/lucene/analysis/PorterStemmer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/PorterStemmer.java (working copy)
@@ -44,8 +44,13 @@
*/
-import java.io.*;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.FileInputStream;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_CHAR;
+import org.apache.lucene.util.ArrayUtil;
+
/**
*
* Stemmer, implementing the Porter Stemming Algorithm
@@ -61,11 +66,10 @@
private int i, /* offset into b */
j, k, k0;
private boolean dirty = false;
- private static final int INC = 50; /* unit of size whereby b is increased */
- private static final int EXTRA = 1;
+ private static final int INITIAL_SIZE = 50;
public PorterStemmer() {
- b = new char[INC];
+ b = new char[INITIAL_SIZE];
i = 0;
}
@@ -81,10 +85,8 @@
* adding characters, you can call stem(void) to process the word.
*/
public void add(char ch) {
- if (b.length <= i + EXTRA) {
- char[] new_b = new char[b.length+INC];
- System.arraycopy(b, 0, new_b, 0, b.length);
- b = new_b;
+ if (b.length <= i) {
+ b = ArrayUtil.grow(b, i+1);
}
b[i++] = ch;
}
@@ -451,8 +453,7 @@
public boolean stem(char[] wordBuffer, int offset, int wordLen) {
reset();
if (b.length < wordLen) {
- char[] new_b = new char[wordLen + EXTRA];
- b = new_b;
+ b = new char[ArrayUtil.oversize(wordLen, NUM_BYTES_CHAR)];
}
System.arraycopy(wordBuffer, offset, b, 0, wordLen);
i = wordLen;
Index: src/java/org/apache/lucene/analysis/PorterStemFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/PorterStemFilter.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/PorterStemFilter.java (working copy)
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
/** Transforms the token stream as per the Porter stemming algorithm.
@@ -38,15 +39,23 @@
}
}
+
+ Note: This filter is aware of the {@link KeywordAttribute}. To prevent
+ certain terms from being passed to the stemmer
+ {@link KeywordAttribute#isKeyword()} should be set to true
+ in a previous {@link TokenStream}.
+
*/
public final class PorterStemFilter extends TokenFilter {
- private PorterStemmer stemmer;
- private TermAttribute termAtt;
+ private final PorterStemmer stemmer;
+ private final TermAttribute termAtt;
+ private final KeywordAttribute keywordAttr;
public PorterStemFilter(TokenStream in) {
super(in);
stemmer = new PorterStemmer();
termAtt = addAttribute(TermAttribute.class);
+ keywordAttr = addAttribute(KeywordAttribute.class);
}
@Override
@@ -54,7 +63,7 @@
if (!input.incrementToken())
return false;
- if (stemmer.stem(termAtt.termBuffer(), 0, termAtt.termLength()))
+ if ((!keywordAttr.isKeyword()) && stemmer.stem(termAtt.termBuffer(), 0, termAtt.termLength()))
termAtt.setTermBuffer(stemmer.getResultBuffer(), 0, stemmer.getResultLength());
return true;
}
Property changes on: src\java\org\apache\lucene\analysis\Tokenizer.java
___________________________________________________________________
Added: svn:mergeinfo
Merged /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:r909334
Index: src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java (working copy)
@@ -20,6 +20,7 @@
import java.io.Reader;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
/**
* LowerCaseTokenizer performs the function of LetterTokenizer
@@ -30,27 +31,98 @@
*
* Note: this does a decent job for most European languages, but does a terrible
* job for some Asian languages, where words are not separated by spaces.
+ *
+ *
+ *
+ * You must specify the required {@link Version} compatibility when creating
+ * {@link LowerCaseTokenizer}:
+ *
+ * - As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
+ * detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
+ * {@link CharTokenizer#normalize(int)} for details.
+ *
+ *
*/
public final class LowerCaseTokenizer extends LetterTokenizer {
- /** Construct a new LowerCaseTokenizer. */
+
+ /**
+ * Construct a new LowerCaseTokenizer.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ *
+ * @param in
+ * the input to split up into tokens
+ */
+ public LowerCaseTokenizer(Version matchVersion, Reader in) {
+ super(matchVersion, in);
+ }
+
+ /**
+ * Construct a new LowerCaseTokenizer using a given {@link AttributeSource}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param source
+ * the attribute source to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public LowerCaseTokenizer(Version matchVersion, AttributeSource source, Reader in) {
+ super(matchVersion, source, in);
+ }
+
+ /**
+ * Construct a new LowerCaseTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param factory
+ * the attribute factory to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public LowerCaseTokenizer(Version matchVersion, AttributeFactory factory, Reader in) {
+ super(matchVersion, factory, in);
+ }
+
+ /**
+ * Construct a new LowerCaseTokenizer.
+ *
+ * @deprecated use {@link #LowerCaseTokenizer(Reader)} instead. This will be
+ * removed in Lucene 4.0.
+ */
+ @Deprecated
public LowerCaseTokenizer(Reader in) {
- super(in);
+ super(Version.LUCENE_30, in);
}
- /** Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. */
+ /**
+ * Construct a new LowerCaseTokenizer using a given {@link AttributeSource}.
+ *
+ * @deprecated use {@link #LowerCaseTokenizer(AttributeSource, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
public LowerCaseTokenizer(AttributeSource source, Reader in) {
- super(source, in);
+ super(Version.LUCENE_30, source, in);
}
- /** Construct a new LowerCaseTokenizer using a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. */
+ /**
+ * Construct a new LowerCaseTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @deprecated use {@link #LowerCaseTokenizer(AttributeSource.AttributeFactory, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
public LowerCaseTokenizer(AttributeFactory factory, Reader in) {
- super(factory, in);
+ super(Version.LUCENE_30, factory, in);
}
/** Converts char to lower case
- * {@link Character#toLowerCase(char)}.*/
+ * {@link Character#toLowerCase(int)}.*/
@Override
- protected char normalize(char c) {
+ protected int normalize(int c) {
return Character.toLowerCase(c);
}
}
Index: src/java/org/apache/lucene/analysis/Token.java
===================================================================
--- src/java/org/apache/lucene/analysis/Token.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/Token.java (working copy)
@@ -29,6 +29,7 @@
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.RamUsageEstimator;
/**
A Token is an occurrence of a term from the text of a field. It consists of
@@ -347,12 +348,12 @@
public char[] resizeTermBuffer(int newSize) {
if (termBuffer == null) {
// The buffer is always at least MIN_BUFFER_SIZE
- termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
} else {
if(termBuffer.length < newSize){
// Not big enough; create a new array with slight
// over allocation and preserve content
- final char[] newCharBuffer = new char[ArrayUtil.getNextSize(newSize)];
+ final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
termBuffer = newCharBuffer;
}
@@ -367,19 +368,19 @@
private void growTermBuffer(int newSize) {
if (termBuffer == null) {
// The buffer is always at least MIN_BUFFER_SIZE
- termBuffer = new char[ArrayUtil.getNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
} else {
if(termBuffer.length < newSize){
// Not big enough; create a new array with slight
// over allocation:
- termBuffer = new char[ArrayUtil.getNextSize(newSize)];
+ termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
}
}
}
private void initTermBuffer() {
if (termBuffer == null) {
- termBuffer = new char[ArrayUtil.getNextSize(MIN_BUFFER_SIZE)];
+ termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
termLength = 0;
}
}
@@ -453,14 +454,14 @@
}
/**
- * EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
*
*
* Get the bitset for any bits that have been set. This is completely distinct from {@link #type()}, although they do share similar purposes.
* The flags can be used to encode information about the token for use by other {@link org.apache.lucene.analysis.TokenFilter}s.
*
- *
+ *
* @return The bits
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
*/
public int getFlags() {
return flags;
Index: src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java (working copy)
@@ -20,30 +20,102 @@
import java.io.Reader;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
-/** A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
- * Adjacent sequences of non-Whitespace characters form tokens. */
+/**
+ * A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens.
+ *
+ * You must specify the required {@link Version} compatibility when creating
+ * {@link WhitespaceTokenizer}:
+ *
+ * - As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
+ * detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
+ * {@link CharTokenizer#normalize(int)} for details.
+ *
+ */
+public class WhitespaceTokenizer extends CharTokenizer {
+
+ /**
+ * Construct a new WhitespaceTokenizer. * @param matchVersion Lucene version
+ * to match See {@link above}
+ *
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, Reader in) {
+ super(matchVersion, in);
+ }
-public class WhitespaceTokenizer extends CharTokenizer {
- /** Construct a new WhitespaceTokenizer. */
+ /**
+ * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param source
+ * the attribute source to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, AttributeSource source, Reader in) {
+ super(matchVersion, source, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @param
+ * matchVersion Lucene version to match See
+ * {@link above}
+ * @param factory
+ * the attribute factory to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, AttributeFactory factory, Reader in) {
+ super(matchVersion, factory, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, Reader)} instead. This will
+ * be removed in Lucene 4.0.
+ */
+ @Deprecated
public WhitespaceTokenizer(Reader in) {
super(in);
}
- /** Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. */
+ /**
+ * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, AttributeSource, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
+ @Deprecated
public WhitespaceTokenizer(AttributeSource source, Reader in) {
super(source, in);
}
- /** Construct a new WhitespaceTokenizer using a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. */
+ /**
+ * Construct a new WhitespaceTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, AttributeSource.AttributeFactory, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
+ @Deprecated
public WhitespaceTokenizer(AttributeFactory factory, Reader in) {
super(factory, in);
}
/** Collects only characters which do not satisfy
- * {@link Character#isWhitespace(char)}.*/
+ * {@link Character#isWhitespace(int)}.*/
@Override
- protected boolean isTokenChar(char c) {
+ protected boolean isTokenChar(int c) {
return !Character.isWhitespace(c);
}
}
Index: src/java/org/apache/lucene/analysis/CharArraySet.java
===================================================================
--- src/java/org/apache/lucene/analysis/CharArraySet.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/CharArraySet.java (working copy)
@@ -1,15 +1,5 @@
package org.apache.lucene.analysis;
-import java.util.Arrays;
-import java.util.AbstractSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Set;
-
-import org.apache.lucene.util.CharacterUtils;
-import org.apache.lucene.util.Version;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,7 +17,14 @@
* limitations under the License.
*/
+import java.util.Arrays;
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+import org.apache.lucene.util.Version;
+
/**
* A simple class that stores Strings as char[]'s in a
* hash table. Note that this is not a general purpose
@@ -58,16 +55,11 @@
* For type safety also {@link #stringIterator()} is provided.
*/
public class CharArraySet extends AbstractSet
+ */
- Note: this does a decent job for most European languages, but does a terrible
- job for some Asian languages, where words are not separated by spaces. */
-
public class LetterTokenizer extends CharTokenizer {
- /** Construct a new LetterTokenizer. */
+
+ /**
+ * Construct a new LetterTokenizer.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param in
+ * the input to split up into tokens
+ */
+ public LetterTokenizer(Version matchVersion, Reader in) {
+ super(matchVersion, in);
+ }
+
+ /**
+ * Construct a new LetterTokenizer using a given {@link AttributeSource}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param source
+ * the attribute source to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public LetterTokenizer(Version matchVersion, AttributeSource source, Reader in) {
+ super(matchVersion, source, in);
+ }
+
+ /**
+ * Construct a new LetterTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param factory
+ * the attribute factory to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public LetterTokenizer(Version matchVersion, AttributeFactory factory, Reader in) {
+ super(matchVersion, factory, in);
+ }
+
+ /**
+ * Construct a new LetterTokenizer.
+ *
+ * @deprecated use {@link #LetterTokenizer(Version, Reader)} instead. This
+ * will be removed in Lucene 4.0.
+ */
public LetterTokenizer(Reader in) {
- super(in);
+ super(Version.LUCENE_30, in);
}
- /** Construct a new LetterTokenizer using a given {@link AttributeSource}. */
+ /**
+ * Construct a new LetterTokenizer using a given {@link AttributeSource}.
+ * @deprecated
+ * use {@link #LetterTokenizer(Version, AttributeSource, Reader)} instead.
+ * This will be removed in Lucene 4.0.
+ */
public LetterTokenizer(AttributeSource source, Reader in) {
- super(source, in);
+ super(Version.LUCENE_30, source, in);
}
- /** Construct a new LetterTokenizer using a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. */
+ /**
+ * Construct a new LetterTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @deprecated use {@link #LetterTokenizer(Version, AttributeSource.AttributeFactory, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
public LetterTokenizer(AttributeFactory factory, Reader in) {
- super(factory, in);
+ super(Version.LUCENE_30, factory, in);
}
-
+
/** Collects only characters which satisfy
- * {@link Character#isLetter(char)}.*/
+ * {@link Character#isLetter(int)}.*/
@Override
- protected boolean isTokenChar(char c) {
+ protected boolean isTokenChar(int c) {
return Character.isLetter(c);
}
}
Index: src/java/org/apache/lucene/analysis/StopAnalyzer.java
===================================================================
--- src/java/org/apache/lucene/analysis/StopAnalyzer.java (revision 916567)
+++ src/java/org/apache/lucene/analysis/StopAnalyzer.java (working copy)
@@ -99,7 +99,7 @@
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
- final Tokenizer source = new LowerCaseTokenizer(reader);
+ final Tokenizer source = new LowerCaseTokenizer(matchVersion, reader);
return new TokenStreamComponents(source, new StopFilter(matchVersion,
source, stopwords));
}
Index: src/java/org/apache/lucene/search/FieldCache.java
===================================================================
--- src/java/org/apache/lucene/search/FieldCache.java (revision 916567)
+++ src/java/org/apache/lucene/search/FieldCache.java (working copy)
@@ -516,12 +516,7 @@
/**
* EXPERT: A unique Identifier/Description for each item in the FieldCache.
* Can be useful for logging/debugging.
- *
- * EXPERIMENTAL API: This API is considered extremely advanced
- * and experimental. It may be removed or altered w/o warning in future
- * releases
- * of Lucene.
- *
+ * @lucene.experimental
*/
public static abstract class CacheEntry {
public abstract Object getReaderKey();
@@ -586,12 +581,7 @@
* associated with it has garbage collected will prevent the Value itself
* from being garbage collected when the Cache drops the WeakReference.
*
- *
- * EXPERIMENTAL API: This API is considered extremely advanced
- * and experimental. It may be removed or altered w/o warning in future
- * releases
- * of Lucene.
- *
+ * @lucene.experimental
*/
public abstract CacheEntry[] getCacheEntries();
@@ -604,12 +594,7 @@
* It should not be relied on for "Cache maintenance" in general
* application code.
*
- *
- * EXPERIMENTAL API: This API is considered extremely advanced
- * and experimental. It may be removed or altered w/o warning in future
- * releases
- * of Lucene.
- *
+ * @lucene.experimental
*/
public abstract void purgeAllCaches();
Index: src/java/org/apache/lucene/search/MultiTermQuery.java
===================================================================
--- src/java/org/apache/lucene/search/MultiTermQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/MultiTermQuery.java (working copy)
@@ -55,7 +55,7 @@
* computing unhelpful scores, and it tries to pick the most
* performant rewrite method given the query. If you
* need scoring (like {@link FuzzyQuery}, use
- * {@link #TOP_TERMS_SCORING_BOOLEAN_REWRITE} which uses
+ * {@link TopTermsScoringBooleanQueryRewrite} which uses
* a priority queue to only collect competitive terms
* and not hit this limitation.
*
@@ -268,10 +268,41 @@
* @see #setRewriteMethod */
public final static RewriteMethod SCORING_BOOLEAN_QUERY_REWRITE = new ScoringBooleanQueryRewrite();
- private static final class TopTermsScoringBooleanQueryRewrite extends BooleanQueryRewrite {
+ /** A rewrite method that first translates each term into
+ * {@link BooleanClause.Occur#SHOULD} clause in a
+ * BooleanQuery, and keeps the scores as computed by the
+ * query.
+ *
+ * This rewrite mode only uses the top scoring terms
+ * so it will not overflow the boolean max clause count.
+ * It is the default rewrite mode for {@link FuzzyQuery}.
+ *
+ * @see #setRewriteMethod */
+ public static final class TopTermsScoringBooleanQueryRewrite extends BooleanQueryRewrite {
+ private final int size;
+
+ /**
+ * Create a TopTermsScoringBooleanQueryRewrite for
+ * at most size terms.
+ *
+ * NOTE: if {@link BooleanQuery#getMaxClauseCount} is smaller than
+ * size, then it will be used instead.
+ */
+ public TopTermsScoringBooleanQueryRewrite(int size) {
+ this.size = size;
+ }
+
+ /**
+ * Create a TopTermsScoringBooleanQueryRewrite that is limited
+ * to at most {@link BooleanQuery#getMaxClauseCount} terms.
+ */
+ public TopTermsScoringBooleanQueryRewrite() {
+ this(Integer.MAX_VALUE);
+ }
+
@Override
public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
- final int maxSize = BooleanQuery.getMaxClauseCount();
+ final int maxSize = Math.min(size, BooleanQuery.getMaxClauseCount());
final PriorityQueue stQueue = new PriorityQueue();
collectTerms(reader, query, new TermCollector() {
public boolean collect(Term t, float boost) {
@@ -301,10 +332,23 @@
query.incTotalNumberOfTerms(bq.clauses().size());
return bq;
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 17;
+ int result = 1;
+ result = prime * result + size;
+ return result;
+ }
- // Make sure we are still a singleton even after deserializing
- protected Object readResolve() {
- return TOP_TERMS_SCORING_BOOLEAN_REWRITE;
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ TopTermsScoringBooleanQueryRewrite other = (TopTermsScoringBooleanQueryRewrite) obj;
+ if (size != other.size) return false;
+ return true;
}
private static class ScoreTerm implements Comparable {
@@ -319,18 +363,6 @@
}
}
}
-
- /** A rewrite method that first translates each term into
- * {@link BooleanClause.Occur#SHOULD} clause in a
- * BooleanQuery, and keeps the scores as computed by the
- * query.
- *
- * This rewrite mode only uses the top scoring terms
- * so it will not overflow the boolean max clause count.
- * It is the default rewrite mode for {@link FuzzyQuery}.
- *
- * @see #setRewriteMethod */
- public final static RewriteMethod TOP_TERMS_SCORING_BOOLEAN_REWRITE = new TopTermsScoringBooleanQueryRewrite();
private static class ConstantScoreBooleanQueryRewrite extends ScoringBooleanQueryRewrite implements Serializable {
@Override
Index: src/java/org/apache/lucene/search/FuzzyQuery.java
===================================================================
--- src/java/org/apache/lucene/search/FuzzyQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/FuzzyQuery.java (working copy)
@@ -31,7 +31,7 @@
* length of 0 - in this case, *every* term will be enumerated and
* cause an edit score calculation.
*
- *
This query uses {@link MultiTermQuery#TOP_TERMS_SCORING_BOOLEAN_REWRITE}
+ *
This query uses {@link MultiTermQuery.TopTermsScoringBooleanQueryRewrite}
* as default. So terms will be collected and scored according to their
* edit distance. Only the top terms are used for building the {@link BooleanQuery}.
* It is not recommended to change the rewrite mode for fuzzy queries.
@@ -40,6 +40,7 @@
public final static float defaultMinSimilarity = 0.5f;
public final static int defaultPrefixLength = 0;
+ public final static int defaultMaxExpansions = Integer.MAX_VALUE;
private float minimumSimilarity;
private int prefixLength;
@@ -60,13 +61,16 @@
* as the query term is considered similar to the query term if the edit distance
* between both terms is less than length(term)*0.5
* @param prefixLength length of common (non-fuzzy) prefix
+ * @param maxExpansions the maximum number of terms to match. If this number is
+ * greater than {@link BooleanQuery#getMaxClauseCount} when the query is rewritten,
+ * then the maxClauseCount will be used instead.
* @throws IllegalArgumentException if minimumSimilarity is >= 1 or < 0
* or if prefixLength < 0
*/
- public FuzzyQuery(Term term, float minimumSimilarity, int prefixLength) throws IllegalArgumentException {
+ public FuzzyQuery(Term term, float minimumSimilarity, int prefixLength,
+ int maxExpansions) {
super(term.field());
this.term = term;
- setRewriteMethod(TOP_TERMS_SCORING_BOOLEAN_REWRITE);
if (minimumSimilarity >= 1.0f)
throw new IllegalArgumentException("minimumSimilarity >= 1");
@@ -74,7 +78,11 @@
throw new IllegalArgumentException("minimumSimilarity < 0");
if (prefixLength < 0)
throw new IllegalArgumentException("prefixLength < 0");
+ if (maxExpansions < 0)
+ throw new IllegalArgumentException("maxExpansions < 0");
+ setRewriteMethod(new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(maxExpansions));
+
if (term.text().length() > 1.0f / (1.0f - minimumSimilarity)) {
this.termLongEnough = true;
}
@@ -84,17 +92,24 @@
}
/**
- * Calls {@link #FuzzyQuery(Term, float) FuzzyQuery(term, minimumSimilarity, 0)}.
+ * Calls {@link #FuzzyQuery(Term, float) FuzzyQuery(term, minimumSimilarity, prefixLength, Integer.MAX_VALUE)}.
*/
- public FuzzyQuery(Term term, float minimumSimilarity) throws IllegalArgumentException {
- this(term, minimumSimilarity, defaultPrefixLength);
+ public FuzzyQuery(Term term, float minimumSimilarity, int prefixLength) {
+ this(term, minimumSimilarity, prefixLength, defaultMaxExpansions);
}
+
+ /**
+ * Calls {@link #FuzzyQuery(Term, float) FuzzyQuery(term, minimumSimilarity, 0, Integer.MAX_VALUE)}.
+ */
+ public FuzzyQuery(Term term, float minimumSimilarity) {
+ this(term, minimumSimilarity, defaultPrefixLength, defaultMaxExpansions);
+ }
/**
- * Calls {@link #FuzzyQuery(Term, float) FuzzyQuery(term, 0.5f, 0)}.
+ * Calls {@link #FuzzyQuery(Term, float) FuzzyQuery(term, 0.5f, 0, Integer.MAX_VALUE)}.
*/
public FuzzyQuery(Term term) {
- this(term, defaultMinSimilarity, defaultPrefixLength);
+ this(term, defaultMinSimilarity, defaultPrefixLength, defaultMaxExpansions);
}
/**
Property changes on: src\java\org\apache\lucene\search\MultiTermQueryWrapperFilter.java
___________________________________________________________________
Added: svn:mergeinfo
Merged /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:r909334
Index: src/java/org/apache/lucene/search/NumericRangeFilter.java
===================================================================
--- src/java/org/apache/lucene/search/NumericRangeFilter.java (revision 916567)
+++ src/java/org/apache/lucene/search/NumericRangeFilter.java (working copy)
@@ -39,9 +39,7 @@
* See {@link NumericRangeQuery} for details on how Lucene
* indexes and searches numeric valued fields.
*
- *
NOTE: This API is experimental and
- * might change in incompatible ways in the next
- * release.
+ * @lucene.experimental
*
* @since 2.9
**/
Index: src/java/org/apache/lucene/search/Filter.java
===================================================================
--- src/java/org/apache/lucene/search/Filter.java (revision 916567)
+++ src/java/org/apache/lucene/search/Filter.java (working copy)
@@ -23,21 +23,20 @@
import org.apache.lucene.util.DocIdBitSet;
/**
- * Abstract base class providing a mechanism to use a subset of an index
- * for restriction or permission of index search results.
- *
+ * Abstract base class for restricting which documents may
+ * be returned during searching.
*/
public abstract class Filter implements java.io.Serializable {
/**
- * Creates a {@link DocIdSet} that provides the documents which should be
- * permitted or prohibited in search results. NOTE: null can be
- * returned if no documents will be accepted by this Filter.
+ * Creates a {@link DocIdSet} enumerating the documents that should be
+ * permitted in search results. NOTE: null can be
+ * returned if no documents are accepted by this Filter.
*
- * Note: This method might be called more than once during a search if the
- * index has more than one segment. In such a case the {@link DocIdSet}
- * must be relative to the document base of the given reader. Yet, the
- * segment readers are passed in increasing document base order.
+ * Note: This method will be called once per segment in
+ * the index during searching. The returned {@link DocIdSet}
+ * must refer to document IDs for that segment, not for
+ * the top-level reader.
*
* @param reader a {@link IndexReader} instance opened on the index currently
* searched on. Note, it is likely that the provided reader does not
Index: src/java/org/apache/lucene/search/Similarity.java
===================================================================
--- src/java/org/apache/lucene/search/Similarity.java (revision 916567)
+++ src/java/org/apache/lucene/search/Similarity.java (working copy)
@@ -602,8 +602,7 @@
* {@link FieldInvertState#getLength()} as the second argument, and
* then multiplies this value by {@link FieldInvertState#getBoost()}.
*
- * WARNING: This API is new and experimental and may
- * suddenly change.
+ * @lucene.experimental
*
* @param field field name
* @param state current processing state for this field
Index: src/java/org/apache/lucene/search/FieldValueHitQueue.java
===================================================================
--- src/java/org/apache/lucene/search/FieldValueHitQueue.java (revision 916567)
+++ src/java/org/apache/lucene/search/FieldValueHitQueue.java (working copy)
@@ -26,11 +26,8 @@
* Uses FieldCache.DEFAULT for maintaining
* internal term lookup tables.
*
- * NOTE: This API is experimental and might change in
- * incompatible ways in the next release.
- *
+ * @lucene.experimental
* @since 2.9
- * @version $Id:
* @see Searcher#search(Query,Filter,int,Sort)
* @see FieldCache
*/
Index: src/java/org/apache/lucene/search/TopScoreDocCollector.java
===================================================================
--- src/java/org/apache/lucene/search/TopScoreDocCollector.java (revision 916567)
+++ src/java/org/apache/lucene/search/TopScoreDocCollector.java (working copy)
@@ -29,10 +29,10 @@
* instance of this collector you should know in advance whether documents are
* going to be collected in doc Id order or not.
*
- * NOTE: The values Float.Nan,
- * Float.NEGATIVE_INFINITY and Float.POSITIVE_INFINITY are
- * not valid scores. This collector will not properly
- * collect hits with such scores.
+ *
NOTE: The values {@link Float#NaN} and
+ * {Float#NEGATIVE_INFINITY} are not valid scores. This
+ * collector will not properly collect hits with such
+ * scores.
*/
public abstract class TopScoreDocCollector extends TopDocsCollector {
@@ -45,6 +45,11 @@
@Override
public void collect(int doc) throws IOException {
float score = scorer.score();
+
+ // This collector cannot handle these scores:
+ assert score != Float.NEGATIVE_INFINITY;
+ assert !Float.isNaN(score);
+
totalHits++;
if (score <= pqTop.score) {
// Since docs are returned in-order (i.e., increasing doc Id), a document
@@ -72,6 +77,10 @@
@Override
public void collect(int doc) throws IOException {
float score = scorer.score();
+
+ // This collector cannot handle NaN
+ assert !Float.isNaN(score);
+
totalHits++;
doc += docBase;
if (score < pqTop.score || (score == pqTop.score && doc > pqTop.doc)) {
Index: src/java/org/apache/lucene/search/payloads/PayloadFunction.java
===================================================================
--- src/java/org/apache/lucene/search/payloads/PayloadFunction.java (revision 916567)
+++ src/java/org/apache/lucene/search/payloads/PayloadFunction.java (working copy)
@@ -18,16 +18,15 @@
import java.io.Serializable;
-
/**
- * An abstract class that defines a way for Payload*Query instances
- * to transform the cumulative effects of payload scores for a document.
- *
+ * An abstract class that defines a way for Payload*Query instances to transform
+ * the cumulative effects of payload scores for a document.
+ *
* @see org.apache.lucene.search.payloads.PayloadTermQuery for more information
- *
- *
- * This class and its derivations are experimental and subject to change
- *
+ *
+ * @lucene.experimental This class and its derivations are experimental and subject to
+ * change
+ *
**/
public abstract class PayloadFunction implements Serializable {
Index: src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java
===================================================================
--- src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (revision 916567)
+++ src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (working copy)
@@ -23,9 +23,13 @@
**/
public class MinPayloadFunction extends PayloadFunction {
- @Override
- public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) {
- return Math.min(currentPayloadScore, currentScore);
+ @Override
+ public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) {
+ if (numPayloadsSeen == 0) {
+ return currentPayloadScore;
+ } else {
+ return Math.min(currentPayloadScore, currentScore);
+ }
}
@Override
Index: src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java
===================================================================
--- src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (revision 916567)
+++ src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (working copy)
@@ -27,7 +27,11 @@
public class MaxPayloadFunction extends PayloadFunction {
@Override
public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) {
- return Math.max(currentPayloadScore, currentScore);
+ if (numPayloadsSeen == 0) {
+ return currentPayloadScore;
+ } else {
+ return Math.max(currentPayloadScore, currentScore);
+ }
}
@Override
Index: src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
===================================================================
--- src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (revision 916567)
+++ src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (working copy)
@@ -44,11 +44,7 @@
* Operates like Highlighter - IndexReader should only contain doc of interest,
* best to use MemoryIndex.
*
- *
- *
- * WARNING: The status of the Payloads feature is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*/
public class PayloadSpanUtil {
Index: src/java/org/apache/lucene/search/SpanQueryFilter.java
===================================================================
--- src/java/org/apache/lucene/search/SpanQueryFilter.java (revision 916567)
+++ src/java/org/apache/lucene/search/SpanQueryFilter.java (working copy)
@@ -34,9 +34,6 @@
*
* This filter does not cache. See the {@link org.apache.lucene.search.CachingSpanFilter} for a wrapper that
* caches.
- *
- *
- * @version $Id:$
*/
public class SpanQueryFilter extends SpanFilter {
protected SpanQuery query;
Index: src/java/org/apache/lucene/search/spans/Spans.java
===================================================================
--- src/java/org/apache/lucene/search/spans/Spans.java (revision 916567)
+++ src/java/org/apache/lucene/search/spans/Spans.java (working copy)
@@ -66,10 +66,7 @@
*
* Note that the return type is a collection, thus the ordering should not be relied upon.
*
- *
- * WARNING: The status of the Payloads feature is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
* @return a List of byte arrays containing the data of this payload, otherwise null if isPayloadAvailable is false
* @throws java.io.IOException
Index: src/java/org/apache/lucene/search/FuzzyTermEnum.java
===================================================================
--- src/java/org/apache/lucene/search/FuzzyTermEnum.java (revision 916567)
+++ src/java/org/apache/lucene/search/FuzzyTermEnum.java (working copy)
@@ -193,7 +193,7 @@
* @return the similarity, 0.0 or less indicates that it matches less than the required
* threshold and 1.0 indicates that the text and target are identical
*/
- private synchronized final float similarity(final String target) {
+ private float similarity(final String target) {
final int m = target.length();
final int n = text.length();
if (n == 0) {
Index: src/java/org/apache/lucene/search/SpanFilterResult.java
===================================================================
--- src/java/org/apache/lucene/search/SpanFilterResult.java (revision 916567)
+++ src/java/org/apache/lucene/search/SpanFilterResult.java (working copy)
@@ -23,8 +23,7 @@
/**
* The results of a SpanQueryFilter. Wraps the BitSet and the position information from the SpanQuery
*
- *
- * NOTE: This API is still experimental and subject to change.
+ * @lucene.experimental
*
**/
public class SpanFilterResult {
Index: src/java/org/apache/lucene/search/Collector.java
===================================================================
--- src/java/org/apache/lucene/search/Collector.java (revision 916567)
+++ src/java/org/apache/lucene/search/Collector.java (working copy)
@@ -116,8 +116,7 @@
* you need that behavior, use {@link
* PositiveScoresOnlyCollector}.
*
- * NOTE: This API is experimental and might change
- * in incompatible ways in the next release.
+ * @lucene.experimental
*
* @since 2.9
*/
Index: src/java/org/apache/lucene/search/function/FieldScoreQuery.java
===================================================================
--- src/java/org/apache/lucene/search/function/FieldScoreQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/FieldScoreQuery.java (working copy)
@@ -52,10 +52,7 @@
* otherwise, for instance if for each query a new index reader is opened, large penalties would be
* paid for loading the field values into memory over and over again!
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*/
public class FieldScoreQuery extends ValueSourceQuery {
Index: src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (working copy)
@@ -40,10 +40,7 @@
* when other documents are inserted or deleted,
* or if a MultiSearcher is used.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*
NOTE: with the switch in 2.9 to segment-based
* searching, if {@link #getValues} is invoked with a
Index: src/java/org/apache/lucene/search/function/OrdFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/OrdFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/OrdFieldSource.java (working copy)
@@ -39,10 +39,7 @@
* when other documents are inserted or deleted,
* or if a MultiSearcher is used.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*
NOTE: with the switch in 2.9 to segment-based
* searching, if {@link #getValues} is invoked with a
Index: src/java/org/apache/lucene/search/function/package.html
===================================================================
--- src/java/org/apache/lucene/search/function/package.html (revision 916567)
+++ src/java/org/apache/lucene/search/function/package.html (working copy)
@@ -26,11 +26,7 @@
The function package provides tight control over documents scores.
-
-WARNING: The status of the search.function package is experimental. The APIs
-introduced here might change in the future and will not be supported anymore
-in such a case.
-
+@lucene.experimental
Two types of queries are available in this package:
Index: src/java/org/apache/lucene/search/function/IntFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/IntFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/IntFieldSource.java (working copy)
@@ -29,10 +29,7 @@
* using
getInts() and makes those values
* available as other numeric types, casting as needed.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
* @see org.apache.lucene.search.function.FieldCacheSource for requirements
* on the field.
Index: src/java/org/apache/lucene/search/function/ValueSourceQuery.java
===================================================================
--- src/java/org/apache/lucene/search/function/ValueSourceQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/ValueSourceQuery.java (working copy)
@@ -37,10 +37,7 @@
*
* Score is set as: Score(doc,query) = query.getBoost()2 * valueSource(doc).
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*/
public class ValueSourceQuery extends Query {
ValueSource valSrc;
Index: src/java/org/apache/lucene/search/function/ShortFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/ShortFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/ShortFieldSource.java (working copy)
@@ -29,10 +29,7 @@
* using getShorts() and makes those values
* available as other numeric types, casting as needed.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
* @see org.apache.lucene.search.function.FieldCacheSource for requirements
* on the field.
Index: src/java/org/apache/lucene/search/function/DocValues.java
===================================================================
--- src/java/org/apache/lucene/search/function/DocValues.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/DocValues.java (working copy)
@@ -25,10 +25,7 @@
* {@link org.apache.lucene.search.function.ValueSource ValueSuorce}
* for a particular field and reader.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*
*/
Index: src/java/org/apache/lucene/search/function/ByteFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/ByteFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/ByteFieldSource.java (working copy)
@@ -29,10 +29,7 @@
* using getBytes() and makes those values
* available as other numeric types, casting as needed.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
* @see org.apache.lucene.search.function.FieldCacheSource for requirements
* on the field.
Index: src/java/org/apache/lucene/search/function/FieldCacheSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/FieldCacheSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/FieldCacheSource.java (working copy)
@@ -34,10 +34,7 @@
*
* Document with no tokens in this field are assigned the Zero value.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*
NOTE: with the switch in 2.9 to segment-based
* searching, if {@link #getValues} is invoked with a
Index: src/java/org/apache/lucene/search/function/FloatFieldSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/FloatFieldSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/FloatFieldSource.java (working copy)
@@ -29,10 +29,7 @@
* using getFloats() and makes those values
* available as other numeric types, casting as needed.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
* @see org.apache.lucene.search.function.FieldCacheSource for requirements
* on the field.
Index: src/java/org/apache/lucene/search/function/CustomScoreQuery.java
===================================================================
--- src/java/org/apache/lucene/search/function/CustomScoreQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/CustomScoreQuery.java (working copy)
@@ -40,12 +40,9 @@
* For most simple/convenient use cases this query is likely to be a
* {@link org.apache.lucene.search.function.FieldScoreQuery FieldScoreQuery}
*
- * Subclasses can modify the computation by overriding {@link #customScore(int, float, float)}.
+ * Subclasses can modify the computation by overriding {@link #getCustomScoreProvider}.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*/
public class CustomScoreQuery extends Query {
@@ -83,7 +80,6 @@
* This parameter is optional - it can be null or even an empty array.
*/
public CustomScoreQuery(Query subQuery, ValueSourceQuery... valSrcQueries) {
- super();
this.subQuery = subQuery;
this.valSrcQueries = valSrcQueries!=null?
valSrcQueries : new ValueSourceQuery[0];
@@ -93,11 +89,23 @@
/*(non-Javadoc) @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader) */
@Override
public Query rewrite(IndexReader reader) throws IOException {
- subQuery = subQuery.rewrite(reader);
+ CustomScoreQuery clone = null;
+
+ final Query sq = subQuery.rewrite(reader);
+ if (sq != subQuery) {
+ clone = (CustomScoreQuery) clone();
+ clone.subQuery = sq;
+ }
+
for(int i = 0; i < valSrcQueries.length; i++) {
- valSrcQueries[i] = (ValueSourceQuery) valSrcQueries[i].rewrite(reader);
+ final ValueSourceQuery v = (ValueSourceQuery) valSrcQueries[i].rewrite(reader);
+ if (v != valSrcQueries[i]) {
+ if (clone == null) clone = (CustomScoreQuery) clone();
+ clone.valSrcQueries[i] = v;
+ }
}
- return this;
+
+ return (clone == null) ? this : clone;
}
/*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(java.util.Set) */
@@ -142,7 +150,8 @@
}
CustomScoreQuery other = (CustomScoreQuery)o;
if (this.getBoost() != other.getBoost() ||
- !this.subQuery.equals(other.subQuery)||
+ !this.subQuery.equals(other.subQuery) ||
+ this.strict != other.strict ||
this.valSrcQueries.length != other.valSrcQueries.length) {
return false;
}
@@ -153,134 +162,19 @@
@Override
public int hashCode() {
return (getClass().hashCode() + subQuery.hashCode() + Arrays.hashCode(valSrcQueries))
- ^ Float.floatToIntBits(getBoost());
- }
+ ^ Float.floatToIntBits(getBoost()) ^ (strict ? 1234 : 4321);
+ }
/**
- * Compute a custom score by the subQuery score and a number of
- * ValueSourceQuery scores.
- *
- * Subclasses can override this method to modify the custom score.
- *
- * If your custom scoring is different than the default herein you
- * should override at least one of the two customScore() methods.
- * If the number of ValueSourceQueries is always < 2 it is
- * sufficient to override the other
- * {@link #customScore(int, float, float) customScore()}
- * method, which is simpler.
- *
- * The default computation herein is a multiplication of given scores:
- *
- * ModifiedScore = valSrcScore * valSrcScores[0] * valSrcScores[1] * ...
- *
- *
- * @param doc id of scored doc.
- * @param subQueryScore score of that doc by the subQuery.
- * @param valSrcScores scores of that doc by the ValueSourceQuery.
- * @return custom score.
+ * Returns a {@link CustomScoreProvider} that calculates the custom scores
+ * for the given {@link IndexReader}. The default implementation returns a default
+ * implementation as specified in the docs of {@link CustomScoreProvider}.
+ * @since 2.9.2
*/
- public float customScore(int doc, float subQueryScore, float valSrcScores[]) {
- if (valSrcScores.length == 1) {
- return customScore(doc, subQueryScore, valSrcScores[0]);
- }
- if (valSrcScores.length == 0) {
- return customScore(doc, subQueryScore, 1);
- }
- float score = subQueryScore;
- for(int i = 0; i < valSrcScores.length; i++) {
- score *= valSrcScores[i];
- }
- return score;
+ protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) throws IOException {
+ return new CustomScoreProvider(reader);
}
- /**
- * Compute a custom score by the subQuery score and the ValueSourceQuery score.
- *
- * Subclasses can override this method to modify the custom score.
- *
- * If your custom scoring is different than the default herein you
- * should override at least one of the two customScore() methods.
- * If the number of ValueSourceQueries is always < 2 it is
- * sufficient to override this customScore() method, which is simpler.
- *
- * The default computation herein is a multiplication of the two scores:
- *
- * ModifiedScore = subQueryScore * valSrcScore
- *
- *
- *
NOTE: The doc is relative to the current
- * reader, last passed to {@link #setNextReader}.
- *
- * @param doc id of scored doc.
- * @param subQueryScore score of that doc by the subQuery.
- * @param valSrcScore score of that doc by the ValueSourceQuery.
- * @return custom score.
- */
- public float customScore(int doc, float subQueryScore, float valSrcScore) {
- return subQueryScore * valSrcScore;
- }
-
- /**
- * Called when the scoring switches to another reader.
- *
- * @param reader
- * next IndexReader
- */
- public void setNextReader(IndexReader reader) throws IOException {
- }
-
- /**
- * Explain the custom score.
- * Whenever overriding {@link #customScore(int, float, float[])},
- * this method should also be overridden to provide the correct explanation
- * for the part of the custom scoring.
- *
- * @param doc doc being explained.
- * @param subQueryExpl explanation for the sub-query part.
- * @param valSrcExpls explanation for the value source part.
- * @return an explanation for the custom score
- */
- public Explanation customExplain(int doc, Explanation subQueryExpl, Explanation valSrcExpls[]) {
- if (valSrcExpls.length == 1) {
- return customExplain(doc, subQueryExpl, valSrcExpls[0]);
- }
- if (valSrcExpls.length == 0) {
- return subQueryExpl;
- }
- float valSrcScore = 1;
- for (int i = 0; i < valSrcExpls.length; i++) {
- valSrcScore *= valSrcExpls[i].getValue();
- }
- Explanation exp = new Explanation( valSrcScore * subQueryExpl.getValue(), "custom score: product of:");
- exp.addDetail(subQueryExpl);
- for (int i = 0; i < valSrcExpls.length; i++) {
- exp.addDetail(valSrcExpls[i]);
- }
- return exp;
- }
-
- /**
- * Explain the custom score.
- * Whenever overriding {@link #customScore(int, float, float)},
- * this method should also be overridden to provide the correct explanation
- * for the part of the custom scoring.
- *
- * @param doc doc being explained.
- * @param subQueryExpl explanation for the sub-query part.
- * @param valSrcExpl explanation for the value source part.
- * @return an explanation for the custom score
- */
- public Explanation customExplain(int doc, Explanation subQueryExpl, Explanation valSrcExpl) {
- float valSrcScore = 1;
- if (valSrcExpl != null) {
- valSrcScore *= valSrcExpl.getValue();
- }
- Explanation exp = new Explanation( valSrcScore * subQueryExpl.getValue(), "custom score: product of:");
- exp.addDetail(subQueryExpl);
- exp.addDetail(valSrcExpl);
- return exp;
- }
-
//=========================== W E I G H T ============================
private class CustomWeight extends Weight {
@@ -374,7 +268,7 @@
for(int i = 0; i < valSrcWeights.length; i++) {
valSrcExpls[i] = valSrcWeights[i].explain(reader, doc);
}
- Explanation customExp = customExplain(doc,subQueryExpl,valSrcExpls);
+ Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(reader).customExplain(doc,subQueryExpl,valSrcExpls);
float sc = getValue() * customExp.getValue();
Explanation res = new ComplexExplanation(
true, sc, CustomScoreQuery.this.toString() + ", product of:");
@@ -401,6 +295,7 @@
private Scorer subQueryScorer;
private Scorer[] valSrcScorers;
private IndexReader reader;
+ private final CustomScoreProvider provider;
private float vScores[]; // reused in score() to avoid allocating this array for each doc
// constructor
@@ -412,7 +307,7 @@
this.valSrcScorers = valSrcScorers;
this.reader = reader;
this.vScores = new float[valSrcScorers.length];
- setNextReader(reader);
+ this.provider = CustomScoreQuery.this.getCustomScoreProvider(reader);
}
@Override
@@ -437,7 +332,7 @@
for (int i = 0; i < valSrcScorers.length; i++) {
vScores[i] = valSrcScorers[i].score();
}
- return qWeight * customScore(subQueryScorer.docID(), subQueryScorer.score(), vScores);
+ return qWeight * provider.customScore(subQueryScorer.docID(), subQueryScorer.score(), vScores);
}
@Override
Index: src/java/org/apache/lucene/search/function/ValueSource.java
===================================================================
--- src/java/org/apache/lucene/search/function/ValueSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/function/ValueSource.java (working copy)
@@ -31,10 +31,7 @@
*
ValueSource implementations differ in RAM requirements: it would always be a factor
* of the number of documents, but for each document the number of bytes can be 1, 2, 4, or 8.
*
- *
- * WARNING: The status of the search.function package is experimental.
- * The APIs introduced here might change in the future and will not be
- * supported anymore in such a case.
+ * @lucene.experimental
*
*
*/
Index: src/java/org/apache/lucene/search/NumericRangeQuery.java
===================================================================
--- src/java/org/apache/lucene/search/NumericRangeQuery.java (revision 916567)
+++ src/java/org/apache/lucene/search/NumericRangeQuery.java (working copy)
@@ -74,8 +74,7 @@
* BooleanQuery rewrite methods without changing
* BooleanQuery's default max clause count.
*
- *
NOTE: This API is experimental and
- * might change in incompatible ways in the next release.
+ * @lucene.experimental
*
*
How it works
*
Index: src/java/org/apache/lucene/search/Query.java
===================================================================
--- src/java/org/apache/lucene/search/Query.java (revision 916567)
+++ src/java/org/apache/lucene/search/Query.java (working copy)
@@ -101,6 +101,8 @@
Weight weight = query.createWeight(searcher);
float sum = weight.sumOfSquaredWeights();
float norm = getSimilarity(searcher).queryNorm(sum);
+ if (Float.isInfinite(norm) || Float.isNaN(norm))
+ norm = 1.0f;
weight.normalize(norm);
return weight;
}
Index: src/java/org/apache/lucene/search/FieldComparator.java
===================================================================
--- src/java/org/apache/lucene/search/FieldComparator.java (revision 916567)
+++ src/java/org/apache/lucene/search/FieldComparator.java (working copy)
@@ -79,8 +79,7 @@
* FieldDoc#fields} when returning the top results.
*
*
- *
NOTE: This API is experimental and might change in
- * incompatible ways in the next release.
+ * @lucene.experimental
*/
public abstract class FieldComparator {
Index: src/java/org/apache/lucene/search/SortField.java
===================================================================
--- src/java/org/apache/lucene/search/SortField.java (revision 916567)
+++ src/java/org/apache/lucene/search/SortField.java (working copy)
@@ -365,8 +365,7 @@
/** Returns the {@link FieldComparator} to use for
* sorting.
*
- *
NOTE: This API is experimental and might change in
- * incompatible ways in the next release.
+ * @lucene.experimental
*
* @param numHits number of top hits the queue will store
* @param sortPos position of this SortField within {@link
Index: src/java/org/apache/lucene/search/QueryWrapperFilter.java
===================================================================
--- src/java/org/apache/lucene/search/QueryWrapperFilter.java (revision 916567)
+++ src/java/org/apache/lucene/search/QueryWrapperFilter.java (working copy)
@@ -31,8 +31,6 @@
* QueryFilter that matches, e.g., only documents modified within the last
* week. The QueryFilter and TermRangeQuery would only need to be reconstructed
* once per day.
- *
- * @version $Id:$
*/
public class QueryWrapperFilter extends Filter {
private Query query;
Index: src/java/org/apache/lucene/search/TopFieldCollector.java
===================================================================
--- src/java/org/apache/lucene/search/TopFieldCollector.java (revision 916567)
+++ src/java/org/apache/lucene/search/TopFieldCollector.java (working copy)
@@ -30,8 +30,7 @@
* See the {@link #create(org.apache.lucene.search.Sort, int, boolean, boolean, boolean, boolean)} method
* for instantiating a TopFieldCollector.
*
- *
NOTE: This API is experimental and might change in
- * incompatible ways in the next release.
+ * @lucene.experimental
*/
public abstract class TopFieldCollector extends TopDocsCollector
{
Index: src/java/org/apache/lucene/search/ParallelMultiSearcher.java
===================================================================
--- src/java/org/apache/lucene/search/ParallelMultiSearcher.java (revision 916567)
+++ src/java/org/apache/lucene/search/ParallelMultiSearcher.java (working copy)
@@ -175,8 +175,14 @@
searchables[i].search(weight, filter, hc);
}
}
-
+
@Override
+ public void close() throws IOException {
+ executor.shutdown();
+ super.close();
+ }
+
+ @Override
HashMap createDocFrequencyMap(Set terms) throws IOException {
final Term[] allTermsArray = terms.toArray(new Term[terms.size()]);
final int[] aggregatedDocFreqs = new int[terms.size()];
Index: src/java/org/apache/lucene/search/IndexSearcher.java
===================================================================
--- src/java/org/apache/lucene/search/IndexSearcher.java (revision 916567)
+++ src/java/org/apache/lucene/search/IndexSearcher.java (working copy)
@@ -86,9 +86,7 @@
/** Expert: directly specify the reader, subReaders and
* their docID starts.
*
- * NOTE: This API is experimental and
- * might change in incompatible ways in the next
- * release.
*/
+ * @lucene.experimental */
public IndexSearcher(IndexReader reader, IndexReader[] subReaders, int[] docStarts) {
this.reader = reader;
this.subReaders = subReaders;
Index: src/java/org/apache/lucene/search/FieldCacheImpl.java
===================================================================
--- src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 916567)
+++ src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy)
@@ -61,27 +61,26 @@
caches.put(StringIndex.class, new StringIndexCache(this));
}
- public void purgeAllCaches() {
+ public synchronized void purgeAllCaches() {
init();
}
- public void purge(IndexReader r) {
+ public synchronized void purge(IndexReader r) {
for(Cache c : caches.values()) {
c.purge(r);
}
}
- public CacheEntry[] getCacheEntries() {
+ public synchronized CacheEntry[] getCacheEntries() {
List result = new ArrayList(17);
- for(final Class cacheType: caches.keySet()) {
- Cache cache = caches.get(cacheType);
- for (final Object readerKey : cache.readerCache.keySet()) {
- // we've now materialized a hard ref
-
- // innerKeys was backed by WeakHashMap, sanity check
- // that it wasn't GCed before we made hard ref
- if (null != readerKey && cache.readerCache.containsKey(readerKey)) {
- Map innerCache = cache.readerCache.get(readerKey);
+ for(final Map.Entry,Cache> cacheEntry: caches.entrySet()) {
+ final Cache cache = cacheEntry.getValue();
+ final Class cacheType = cacheEntry.getKey();
+ synchronized(cache.readerCache) {
+ for (final Map.Entry> readerCacheEntry : cache.readerCache.entrySet()) {
+ final Object readerKey = readerCacheEntry.getKey();
+ if (readerKey == null) continue;
+ final Map innerCache = readerCacheEntry.getValue();
for (final Map.Entry mapEntry : innerCache.entrySet()) {
Entry entry = mapEntry.getKey();
result.add(new CacheEntryImpl(readerKey, entry.field,
Index: src/java/org/apache/lucene/search/DefaultSimilarity.java
===================================================================
--- src/java/org/apache/lucene/search/DefaultSimilarity.java (revision 916567)
+++ src/java/org/apache/lucene/search/DefaultSimilarity.java (working copy)
@@ -29,8 +29,7 @@
* FieldInvertState#getLength()} - {@link
* FieldInvertState#getNumOverlap()}.
*
- * WARNING: This API is new and experimental, and may suddenly
- * change.
*/
+ * @lucene.experimental */
@Override
public float computeNorm(String field, FieldInvertState state) {
final int numTerms;
@@ -85,8 +84,7 @@
* norm. By default this is false, meaning overlap
* tokens are counted just like non-overlap tokens.
*
- * WARNING: This API is new and experimental, and may suddenly
- * change.
+ * @lucene.experimental
*
* @see #computeNorm
*/
Index: src/java/org/apache/lucene/search/FieldComparatorSource.java
===================================================================
--- src/java/org/apache/lucene/search/FieldComparatorSource.java (revision 916567)
+++ src/java/org/apache/lucene/search/FieldComparatorSource.java (working copy)
@@ -23,8 +23,7 @@
/**
* Provides a {@link FieldComparator} for custom field sorting.
*
- * NOTE: This API is experimental and might change in
- * incompatible ways in the next release.
+ * @lucene.experimental
*
*/
public abstract class FieldComparatorSource implements Serializable {
Index: src/java/org/apache/lucene/index/NormsWriterPerField.java
===================================================================
--- src/java/org/apache/lucene/index/NormsWriterPerField.java (revision 916567)
+++ src/java/org/apache/lucene/index/NormsWriterPerField.java (working copy)
@@ -63,11 +63,13 @@
@Override
void finish() {
- assert docIDs.length == norms.length;
if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
if (docIDs.length <= upto) {
assert docIDs.length == upto;
docIDs = ArrayUtil.grow(docIDs, 1+upto);
+ }
+ if (norms.length <= upto) {
+ assert norms.length == upto;
norms = ArrayUtil.grow(norms, 1+upto);
}
final float norm = docState.similarity.computeNorm(fieldInfo.name, fieldState);
Index: src/java/org/apache/lucene/index/ByteBlockPool.java
===================================================================
--- src/java/org/apache/lucene/index/ByteBlockPool.java (revision 916567)
+++ src/java/org/apache/lucene/index/ByteBlockPool.java (working copy)
@@ -35,6 +35,8 @@
import java.util.Arrays;
import org.apache.lucene.util.BytesRef;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+import org.apache.lucene.util.ArrayUtil;
final class ByteBlockPool {
@@ -84,7 +86,8 @@
public void nextBuffer() {
if (1+bufferUpto == buffers.length) {
- byte[][] newBuffers = new byte[(int) (buffers.length*1.5)][];
+ byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1,
+ NUM_BYTES_OBJECT_REF)][];
System.arraycopy(buffers, 0, newBuffers, 0, buffers.length);
buffers = newBuffers;
}
Index: src/java/org/apache/lucene/index/SegmentInfo.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentInfo.java (revision 916567)
+++ src/java/org/apache/lucene/index/SegmentInfo.java (working copy)
@@ -34,8 +34,7 @@
* Information about a segment such as it's name, directory, and files related
* to the segment.
*
- * * NOTE: This API is new and still experimental
- * (subject to change suddenly in the next release)
+ * @lucene.experimental
*/
public final class SegmentInfo {
@@ -558,6 +557,7 @@
*/
void write(IndexOutput output)
throws IOException {
+ assert delCount <= docCount: "delCount=" + delCount + " docCount=" + docCount + " segment=" + name;
output.writeString(name);
output.writeInt(docCount);
output.writeLong(delGen);
Index: src/java/org/apache/lucene/index/SegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentReader.java (revision 916567)
+++ src/java/org/apache/lucene/index/SegmentReader.java (working copy)
@@ -49,10 +49,8 @@
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import org.apache.lucene.util.BytesRef;
-/** @version $Id */
/**
- * NOTE: This API is new and still experimental
- * (subject to change suddenly in the next release)
+ * @lucene.experimental
*/
public class SegmentReader extends IndexReader implements Cloneable {
protected boolean readOnly;
@@ -205,7 +203,11 @@
storeCFSReader.close();
}
- // Force FieldCache to evict our entries at this point
+ // Force FieldCache to evict our entries at this
+ // point. If the exception occurred while
+ // initializing the core readers, then
+ // origInstance will be null, and we don't want
+ // to call FieldCache.purge (it leads to NPE):
if (origInstance != null) {
FieldCache.DEFAULT.purge(origInstance);
}
@@ -583,7 +585,7 @@
deletedDocsRef = new AtomicInteger(1);
assert checkDeletedCounts();
if (deletedDocs.size() != si.docCount) {
- throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount);
+ throw new CorruptIndexException("document count mismatch: deleted docs count " + deletedDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name);
}
} else
assert si.getDelCount() == 0;
Index: src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java
===================================================================
--- src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java (revision 916567)
+++ src/java/org/apache/lucene/index/SnapshotDeletionPolicy.java (working copy)
@@ -39,8 +39,7 @@
* snapshot held when a writer is closed will "survive"
* when the next writer is opened.
*
- * WARNING: This API is a new and experimental and
- * may suddenly change.
*/
+ * @lucene.experimental */
public class SnapshotDeletionPolicy implements IndexDeletionPolicy {
Index: src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
===================================================================
--- src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (revision 916567)
+++ src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (working copy)
@@ -24,6 +24,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Gathers all Fieldables for a document under the same
@@ -340,7 +341,7 @@
// enough space to recycle all outstanding PerDoc
// instances
assert allocCount == 1+docFreeList.length;
- docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+ docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
}
return new PerDoc();
} else
Index: src/java/org/apache/lucene/index/DocFieldConsumers.java
===================================================================
--- src/java/org/apache/lucene/index/DocFieldConsumers.java (revision 916567)
+++ src/java/org/apache/lucene/index/DocFieldConsumers.java (working copy)
@@ -25,6 +25,7 @@
import java.io.IOException;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
/** This is just a "splitter" class: it lets you wrap two
* DocFieldConsumer instances as a single consumer. */
@@ -117,7 +118,7 @@
// enough space to recycle all outstanding PerDoc
// instances
assert allocCount == 1+docFreeList.length;
- docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+ docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
}
return new PerDoc();
} else
Index: src/java/org/apache/lucene/index/MergePolicy.java
===================================================================
--- src/java/org/apache/lucene/index/MergePolicy.java (revision 916567)
+++ src/java/org/apache/lucene/index/MergePolicy.java (working copy)
@@ -51,8 +51,7 @@
* The default MergePolicy is {@link
* LogByteSizeMergePolicy}.
*
- * NOTE: This API is new and still experimental
- * (subject to change suddenly in the next release)
+ * @lucene.experimental
*
* NOTE: This class typically requires access to
* package-private APIs (e.g. SegmentInfos) to do its job;
Index: src/java/org/apache/lucene/index/StoredFieldsWriter.java
===================================================================
--- src/java/org/apache/lucene/index/StoredFieldsWriter.java (revision 916567)
+++ src/java/org/apache/lucene/index/StoredFieldsWriter.java (working copy)
@@ -20,6 +20,7 @@
import java.io.IOException;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
/** This is a DocFieldConsumer that writes stored fields. */
final class StoredFieldsWriter {
@@ -108,7 +109,7 @@
// enough space to recycle all outstanding PerDoc
// instances
assert allocCount == 1+docFreeList.length;
- docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+ docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
}
return new PerDoc();
} else
Index: src/java/org/apache/lucene/index/SegmentInfos.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentInfos.java (revision 916567)
+++ src/java/org/apache/lucene/index/SegmentInfos.java (working copy)
@@ -41,8 +41,7 @@
* A collection of segmentInfo objects with methods for operating on
* those segments in relation to the file system.
*
- *
NOTE: This API is new and still experimental
- * (subject to change suddenly in the next release)
+ * @lucene.experimental
*/
public final class SegmentInfos extends Vector {
Index: src/java/org/apache/lucene/index/CheckIndex.java
===================================================================
--- src/java/org/apache/lucene/index/CheckIndex.java (revision 916567)
+++ src/java/org/apache/lucene/index/CheckIndex.java (working copy)
@@ -44,9 +44,7 @@
* As this tool checks every byte in the index, on a large
* index it can take quite a long time to run.
*
- *
WARNING: this tool and API is new and
- * experimental and is subject to suddenly change in the
- * next release. Please make a complete backup of your
+ * @lucene.experimental Please make a complete backup of your
* index before using this to fix your index!
*/
public class CheckIndex {
@@ -57,8 +55,7 @@
/**
* Returned from {@link #checkIndex()} detailing the health and status of the index.
*
- *
WARNING: this API is new and experimental and is
- * subject to suddenly change in the next release.
+ * @lucene.experimental
**/
public static class Status {
Index: src/java/org/apache/lucene/index/MergeScheduler.java
===================================================================
--- src/java/org/apache/lucene/index/MergeScheduler.java (revision 916567)
+++ src/java/org/apache/lucene/index/MergeScheduler.java (working copy)
@@ -24,8 +24,7 @@
* selected by a {@link MergePolicy}. The default
* MergeScheduler is {@link ConcurrentMergeScheduler}.
*
- * NOTE: This API is new and still experimental
- * (subject to change suddenly in the next release)
+ * @lucene.experimental
*
* NOTE: This class typically requires access to
* package-private APIs (eg, SegmentInfos) to do its job;
Index: src/java/org/apache/lucene/index/IndexCommit.java
===================================================================
--- src/java/org/apache/lucene/index/IndexCommit.java (revision 916567)
+++ src/java/org/apache/lucene/index/IndexCommit.java (working copy)
@@ -37,8 +37,7 @@
* associated with it. The segments file associated with a
* later index commit point would have a larger N.
*
- * WARNING: This API is a new and experimental and
- * may suddenly change.
+ * @lucene.experimental
*/
public abstract class IndexCommit {
Index: src/java/org/apache/lucene/index/DirectoryReader.java
===================================================================
--- src/java/org/apache/lucene/index/DirectoryReader.java (revision 916567)
+++ src/java/org/apache/lucene/index/DirectoryReader.java (working copy)
@@ -985,6 +985,12 @@
// not a good idea):
FieldCache.DEFAULT.purge(this);
+ if (writer != null) {
+ // Since we just closed, writer may now be able to
+ // delete unused files:
+ writer.deleteUnusedFiles();
+ }
+
// throw the first exception
if (ioe != null) throw ioe;
}
@@ -1032,7 +1038,7 @@
/**
* Expert: return the IndexCommit that this reader has opened.
*
- * WARNING: this API is new and experimental and may suddenly change.
+ * @lucene.experimental
*/
@Override
public IndexCommit getIndexCommit() throws IOException {
Index: src/java/org/apache/lucene/index/MultipleTermPositions.java
===================================================================
--- src/java/org/apache/lucene/index/MultipleTermPositions.java (revision 916567)
+++ src/java/org/apache/lucene/index/MultipleTermPositions.java (working copy)
@@ -17,14 +17,14 @@
* limitations under the License.
*/
-import org.apache.lucene.util.PriorityQueue;
-
import java.io.IOException;
import java.util.Arrays;
-
import java.util.LinkedList;
import java.util.List;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.ArrayUtil;
+
/**
* Allows you to iterate over the {@link TermPositions} for multiple {@link Term}s as
* a single {@link TermPositions}.
@@ -85,10 +85,8 @@
}
private void growArray() {
- int[] newArray = new int[_arraySize * 2];
- System.arraycopy(_array, 0, newArray, 0, _arraySize);
- _array = newArray;
- _arraySize *= 2;
+ _array = ArrayUtil.grow(_array, _arraySize+1);
+ _arraySize = _array.length;
}
}
Index: src/java/org/apache/lucene/index/TermsHash.java
===================================================================
--- src/java/org/apache/lucene/index/TermsHash.java (revision 916567)
+++ src/java/org/apache/lucene/index/TermsHash.java (working copy)
@@ -26,6 +26,7 @@
import java.io.IOException;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
/** This class implements {@link InvertedDocConsumer}, which
* is passed each token produced by the analyzer on each
@@ -92,7 +93,7 @@
assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ": postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + " consumer=" + consumer;
- final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+ final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != postingsFreeList.length) {
RawPostingList[] newArray = new RawPostingList[newSize];
System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
@@ -225,7 +226,7 @@
if (newPostingsAllocCount > postingsFreeList.length)
// Pre-allocate the postingsFreeList so it's large
// enough to hold all postings we've given out
- postingsFreeList = new RawPostingList[ArrayUtil.getNextSize(newPostingsAllocCount)];
+ postingsFreeList = new RawPostingList[ArrayUtil.oversize(newPostingsAllocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
}
postingsFreeCount -= numToCopy;
Index: src/java/org/apache/lucene/index/FieldInvertState.java
===================================================================
--- src/java/org/apache/lucene/index/FieldInvertState.java (revision 916567)
+++ src/java/org/apache/lucene/index/FieldInvertState.java (working copy)
@@ -23,8 +23,7 @@
* being added to the index. The information collected in this class is
* also used to calculate the normalization factor for a field.
*
- * WARNING: This API is new and experimental, and may suddenly
- * change.
+ * @lucene.experimental
*/
public final class FieldInvertState {
int position;
Index: src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
===================================================================
--- src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (revision 916567)
+++ src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (working copy)
@@ -20,6 +20,7 @@
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
import java.io.IOException;
import java.util.Collection;
@@ -117,7 +118,7 @@
// enough space to recycle all outstanding PerDoc
// instances
assert allocCount == 1+docFreeList.length;
- docFreeList = new PerDoc[ArrayUtil.getNextSize(allocCount)];
+ docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
}
return new PerDoc();
} else
@@ -266,6 +267,8 @@
void addField(final int fieldNumber) {
if (numVectorFields == fieldNumbers.length) {
fieldNumbers = ArrayUtil.grow(fieldNumbers);
+ }
+ if (numVectorFields == fieldPointers.length) {
fieldPointers = ArrayUtil.grow(fieldPointers);
}
fieldNumbers[numVectorFields] = fieldNumber;
Index: src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/IndexReader.java (revision 916567)
+++ src/java/org/apache/lucene/index/IndexReader.java (working copy)
@@ -1266,8 +1266,7 @@
* readers that correspond to a Directory with its own
* segments_N file.
*
- * WARNING: this API is new and experimental and
- * may suddenly change.
+ * @lucene.experimental
*/
public IndexCommit getIndexCommit() throws IOException {
throw new UnsupportedOperationException("This reader does not support this method.");
Index: src/java/org/apache/lucene/index/DocumentsWriter.java
===================================================================
--- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 916567)
+++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy)
@@ -42,6 +42,7 @@
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* This class accepts multiple added documents and directly
@@ -999,57 +1000,58 @@
assert checkDeleteTerm(null);
// Delete by term
- try {
- Fields fields = reader.fields();
- TermsEnum termsEnum = null;
-
- String currentField = null;
- BytesRef termRef = new BytesRef();
- DocsEnum docs = null;
-
- for (Entry entry: deletesFlushed.terms.entrySet()) {
- Term term = entry.getKey();
- // Since we visit terms sorted, we gain performance
- // by re-using the same TermsEnum and seeking only
- // forwards
- if (term.field() != currentField) {
- // nocommit -- once we sync up branch again, add
- // assert that this field is always > last one
- currentField = term.field();
- Terms terms = fields.terms(currentField);
- if (terms != null) {
- termsEnum = terms.iterator();
- } else {
- termsEnum = null;
+ if (deletesFlushed.terms.size() > 0) {
+ try {
+ Fields fields = reader.fields();
+ TermsEnum termsEnum = null;
+
+ String currentField = null;
+ BytesRef termRef = new BytesRef();
+ DocsEnum docs = null;
+
+ for (Entry entry: deletesFlushed.terms.entrySet()) {
+ Term term = entry.getKey();
+ // Since we visit terms sorted, we gain performance
+ // by re-using the same TermsEnum and seeking only
+ // forwards
+ if (term.field() != currentField) {
+ // nocommit -- once we sync up branch again, add
+ // assert that this field is always > last one
+ currentField = term.field();
+ Terms terms = fields.terms(currentField);
+ if (terms != null) {
+ termsEnum = terms.iterator();
+ } else {
+ termsEnum = null;
+ }
}
- }
-
- if (termsEnum == null) {
- continue;
- }
-
- termRef.copy(term.text());
- if (termsEnum.seek(termRef) == TermsEnum.SeekStatus.FOUND) {
- DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
-
- if (docsEnum != null) {
- docs = docsEnum;
- int limit = entry.getValue().getNum();
- while (true) {
- final int docID = docs.nextDoc();
- if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
- break;
+
+ if (termsEnum == null) {
+ continue;
+ }
+
+ termRef.copy(term.text());
+ if (termsEnum.seek(termRef) == TermsEnum.SeekStatus.FOUND) {
+ DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
+
+ if (docsEnum != null) {
+ docs = docsEnum;
+ int limit = entry.getValue().getNum();
+ while (true) {
+ final int docID = docs.nextDoc();
+ if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
+ break;
+ }
+ reader.deleteDocument(docID);
+ any = true;
}
- reader.deleteDocument(docID);
- any = true;
}
}
}
+ } finally {
+ //docs.close();
}
- } finally {
- //docs.close();
}
-
// Delete by docID
for (Integer docIdInt : deletesFlushed.docIDs) {
int docID = docIdInt.intValue();
@@ -1060,23 +1062,28 @@
}
// Delete by query
- IndexSearcher searcher = new IndexSearcher(reader);
- for (Entry entry : deletesFlushed.queries.entrySet()) {
- Query query = entry.getKey();
- int limit = entry.getValue().intValue();
- Weight weight = query.weight(searcher);
- Scorer scorer = weight.scorer(reader, true, false);
- if (scorer != null) {
- while(true) {
- int doc = scorer.nextDoc();
- if (((long) docIDStart) + doc >= limit)
- break;
- reader.deleteDocument(doc);
- any = true;
+ if (deletesFlushed.queries.size() > 0) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ try {
+ for (Entry entry : deletesFlushed.queries.entrySet()) {
+ Query query = entry.getKey();
+ int limit = entry.getValue().intValue();
+ Weight weight = query.weight(searcher);
+ Scorer scorer = weight.scorer(reader, true, false);
+ if (scorer != null) {
+ while(true) {
+ int doc = scorer.nextDoc();
+ if (((long) docIDStart) + doc >= limit)
+ break;
+ reader.deleteDocument(doc);
+ any = true;
+ }
+ }
}
+ } finally {
+ searcher.close();
}
}
- searcher.close();
return any;
}
@@ -1507,7 +1514,7 @@
int gap = doc.docID - nextWriteDocID;
if (gap >= waiting.length) {
// Grow queue
- DocWriter[] newArray = new DocWriter[ArrayUtil.getNextSize(gap)];
+ DocWriter[] newArray = new DocWriter[ArrayUtil.oversize(gap, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert nextWriteLoc >= 0;
System.arraycopy(waiting, nextWriteLoc, newArray, 0, waiting.length-nextWriteLoc);
System.arraycopy(waiting, 0, newArray, waiting.length-nextWriteLoc, nextWriteLoc);
Index: src/java/org/apache/lucene/index/TermVectorsReader.java
===================================================================
--- src/java/org/apache/lucene/index/TermVectorsReader.java (revision 916567)
+++ src/java/org/apache/lucene/index/TermVectorsReader.java (working copy)
@@ -20,6 +20,7 @@
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
import java.io.IOException;
import java.util.Arrays;
@@ -460,18 +461,14 @@
if (preUTF8) {
// Term stored as java chars
if (charBuffer.length < totalLength) {
- char[] newCharBuffer = new char[(int) (1.5*totalLength)];
- System.arraycopy(charBuffer, 0, newCharBuffer, 0, start);
- charBuffer = newCharBuffer;
+ charBuffer = ArrayUtil.grow(charBuffer, totalLength);
}
tvf.readChars(charBuffer, start, deltaLength);
term = new String(charBuffer, 0, totalLength);
} else {
// Term stored as utf8 bytes
if (byteBuffer.length < totalLength) {
- byte[] newByteBuffer = new byte[(int) (1.5*totalLength)];
- System.arraycopy(byteBuffer, 0, newByteBuffer, 0, start);
- byteBuffer = newByteBuffer;
+ byteBuffer = ArrayUtil.grow(byteBuffer, totalLength);
}
tvf.readBytes(byteBuffer, start, deltaLength);
term = new String(byteBuffer, 0, totalLength, "UTF-8");
Index: src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexWriter.java (revision 916567)
+++ src/java/org/apache/lucene/index/IndexWriter.java (working copy)
@@ -362,8 +362,7 @@
* if you attempt to reopen any of those readers, you'll
* hit an {@link AlreadyClosedException}.
*
- * NOTE: This API is experimental and might
- * change in incompatible ways in the next release.
+ * @lucene.experimental
*
* @return IndexReader that covers entire index plus all
* changes made so far by this IndexWriter instance
@@ -3327,13 +3326,19 @@
}
}
- // This is called after pending added and deleted
- // documents have been flushed to the Directory but before
- // the change is committed (new segments_N file written).
- void doAfterFlush()
- throws IOException {
- }
+ /**
+ * A hook for extending classes to execute operations after pending added and
+ * deleted documents have been flushed to the Directory but before the change
+ * is committed (new segments_N file written).
+ */
+ protected void doAfterFlush() throws IOException {}
+ /**
+ * A hook for extending classes to execute operations before pending added and
+ * deleted documents are flushed to the Directory.
+ */
+ protected void doBeforeFlush() throws IOException {}
+
/** Expert: prepare for commit.
*
* NOTE: if this method hits an OutOfMemoryError
@@ -3540,6 +3545,8 @@
assert testPoint("startDoFlush");
+ doBeforeFlush();
+
flushCount++;
// If we are flushing because too many deletes
@@ -4857,8 +4864,7 @@
* search, but will reduce search latency on opening a
* new near real-time reader after a merge completes.
*
- *
NOTE: This API is experimental and might
- * change in incompatible ways in the next release.
+ * @lucene.experimental
*
* NOTE: warm is called before any deletes have
* been carried over to the merged segment. */
@@ -4917,4 +4923,25 @@
synchronized boolean isClosed() {
return closed;
}
+
+ /** Expert: remove any index files that are no longer
+ * used.
+ *
+ *
IndexWriter normally deletes unused files itself,
+ * during indexing. However, on Windows, which disallows
+ * deletion of open files, if there is a reader open on
+ * the index then those files cannot be deleted. This is
+ * fine, because IndexWriter will periodically retry
+ * the deletion.
+ *
+ * However, IndexWriter doesn't try that often: only
+ * on open, close, flushing a new segment, and finishing
+ * a merge. If you don't do any of these actions with your
+ * IndexWriter, you'll see the unused files linger. If
+ * that's a problem, call this method to delete them
+ * (once you've closed the open readers that were
+ * preventing their deletion). */
+ public synchronized void deleteUnusedFiles() throws IOException {
+ deleter.deletePendingFiles();
+ }
}
Index: src/java/org/apache/lucene/index/IndexFileDeleter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 916567)
+++ src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy)
@@ -343,7 +343,7 @@
deletePendingFiles();
}
- private void deletePendingFiles() throws IOException {
+ public void deletePendingFiles() throws IOException {
if (deletable != null) {
List oldDeletable = deletable;
deletable = null;
Index: src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java (revision 916567)
+++ src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java (working copy)
@@ -26,6 +26,7 @@
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
// TODO: we now pulse entirely according to docFreq of the
// term; it might be better to eg pulse by "net bytes used"
@@ -81,7 +82,7 @@
}
void reallocPositions(int minSize) {
- final Position[] newArray = new Position[ArrayUtil.getNextSize(minSize)];
+ final Position[] newArray = new Position[ArrayUtil.oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(positions, 0, newArray, 0, positions.length);
for(int i=positions.length;i bytes.length)
- bytes = new byte[(int) (length*1.25)];
+ if (bytes == null || length > bytes.length) {
+ bytes = new byte[ArrayUtil.oversize(length, 1)];
+ }
readBytes(bytes, 0, length);
return new String(bytes, 0, length, "UTF-8");
}
private String readModifiedUTF8String() throws IOException {
int length = readVInt();
- if (chars == null || length > chars.length)
- chars = new char[length];
+ if (chars == null || length > chars.length) {
+ chars = new char[ArrayUtil.oversize(length, RamUsageEstimator.NUM_BYTES_CHAR)];
+ }
readChars(chars, 0, length);
return new String(chars, 0, length);
}
@@ -157,10 +162,11 @@
else if ((b & 0xE0) != 0xE0) {
buffer[i] = (char)(((b & 0x1F) << 6)
| (readByte() & 0x3F));
- } else
+ } else {
buffer[i] = (char)(((b & 0x0F) << 12)
| ((readByte() & 0x3F) << 6)
| (readByte() & 0x3F));
+ }
}
}
@@ -181,10 +187,9 @@
byte b = readByte();
if ((b & 0x80) == 0){
//do nothing, we only need one byte
- }
- else if ((b & 0xE0) != 0xE0) {
+ } else if ((b & 0xE0) != 0xE0) {
readByte();//read an additional byte
- } else{
+ } else {
//read two additional bytes.
readByte();
readByte();
Index: src/java/org/apache/lucene/store/ChecksumIndexInput.java
===================================================================
--- src/java/org/apache/lucene/store/ChecksumIndexInput.java (revision 916567)
+++ src/java/org/apache/lucene/store/ChecksumIndexInput.java (working copy)
@@ -22,7 +22,10 @@
import java.util.zip.Checksum;
/** Writes bytes through to a primary IndexOutput, computing
- * checksum as it goes. Note that you cannot use seek(). */
+ * checksum as it goes. Note that you cannot use seek().
+ *
+ * @lucene.internal
+ */
public class ChecksumIndexInput extends IndexInput {
IndexInput main;
Checksum digest;
Index: src/java/org/apache/lucene/store/NativeFSLockFactory.java
===================================================================
--- src/java/org/apache/lucene/store/NativeFSLockFactory.java (revision 916567)
+++ src/java/org/apache/lucene/store/NativeFSLockFactory.java (working copy)
@@ -159,12 +159,21 @@
/*
* The javadocs for FileChannel state that you should have
* a single instance of a FileChannel (per JVM) for all
- * locking against a given file. To ensure this, we have
- * a single (static) HashSet that contains the file paths
- * of all currently locked locks. This protects against
- * possible cases where different Directory instances in
- * one JVM (each with their own NativeFSLockFactory
- * instance) have set the same lock dir and lock prefix.
+ * locking against a given file (locks are tracked per
+ * FileChannel instance in Java 1.4/1.5). Even using the same
+ * FileChannel instance is not completely thread-safe with Java
+ * 1.4/1.5 though. To work around this, we have a single (static)
+ * HashSet that contains the file paths of all currently
+ * locked locks. This protects against possible cases
+ * where different Directory instances in one JVM (each
+ * with their own NativeFSLockFactory instance) have set
+ * the same lock dir and lock prefix. However, this will not
+ * work when LockFactorys are created by different
+ * classloaders (eg multiple webapps).
+ *
+ * TODO: Java 1.6 tracks system wide locks in a thread safe manner
+ * (same FileChannel instance or not), so we may want to
+ * change this when Lucene moves to Java 1.6.
*/
private static HashSet LOCK_HELD = new HashSet();
Index: src/java/org/apache/lucene/store/FileSwitchDirectory.java
===================================================================
--- src/java/org/apache/lucene/store/FileSwitchDirectory.java (revision 916567)
+++ src/java/org/apache/lucene/store/FileSwitchDirectory.java (working copy)
@@ -31,8 +31,7 @@
* to this class, and must allow multiple threads to call
* contains at once.
*
- * NOTE: this API is new and experimental and is
- * subject to suddenly change in the next release.
+ * @lucene.experimental
*/
public class FileSwitchDirectory extends Directory {
Index: src/java/org/apache/lucene/util/BitVector.java
===================================================================
--- src/java/org/apache/lucene/util/BitVector.java (revision 916567)
+++ src/java/org/apache/lucene/util/BitVector.java (working copy)
@@ -63,7 +63,7 @@
/** Sets the value of bit to one. */
public final void set(int bit) {
if (bit >= size) {
- throw new ArrayIndexOutOfBoundsException(bit);
+ throw new ArrayIndexOutOfBoundsException("bit=" + bit + " size=" + size);
}
bits[bit >> 3] |= 1 << (bit & 7);
count = -1;
@@ -73,7 +73,7 @@
* returns true if bit was already set */
public final boolean getAndSet(int bit) {
if (bit >= size) {
- throw new ArrayIndexOutOfBoundsException(bit);
+ throw new ArrayIndexOutOfBoundsException("bit=" + bit + " size=" + size);
}
final int pos = bit >> 3;
final int v = bits[pos];
Index: src/java/org/apache/lucene/util/StringHelper.java
===================================================================
--- src/java/org/apache/lucene/util/StringHelper.java (revision 916567)
+++ src/java/org/apache/lucene/util/StringHelper.java (working copy)
@@ -20,6 +20,8 @@
/**
* Methods for manipulating strings.
+ *
+ * @lucene.internal
*/
public abstract class StringHelper {
/**
Index: src/java/org/apache/lucene/util/BitUtil.java
===================================================================
--- src/java/org/apache/lucene/util/BitUtil.java (revision 916567)
+++ src/java/org/apache/lucene/util/BitUtil.java (working copy)
@@ -18,8 +18,7 @@
package org.apache.lucene.util; // from org.apache.solr.util rev 555343
/** A variety of high efficiency bit twiddling routines.
- *
- * @version $Id$
+ * @lucene.internal
*/
public class BitUtil {
Index: src/java/org/apache/lucene/util/CharacterUtils.java
===================================================================
--- src/java/org/apache/lucene/util/CharacterUtils.java (revision 916567)
+++ src/java/org/apache/lucene/util/CharacterUtils.java (working copy)
@@ -1,5 +1,8 @@
package org.apache.lucene.util;
+import java.io.IOException;
+import java.io.Reader;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -21,6 +24,8 @@
* {@link CharacterUtils} provides a unified interface to Character-related
* operations to implement backwards compatible character operations based on a
* {@link Version} instance.
+ *
+ * @lucene.internal
*/
public abstract class CharacterUtils {
private static final Java4CharacterUtils JAVA_4 = new Java4CharacterUtils();
@@ -104,10 +109,52 @@
* the char array.
*/
public abstract int codePointAt(final char[] chars, final int offset, final int limit);
+
+ /**
+ * Creates a new {@link CharacterBuffer} and allocates a char[]
+ * of the given bufferSize.
+ *
+ * @param bufferSize
+ * the internal char buffer size, must be >= 2
+ * @return a new {@link CharacterBuffer} instance.
+ */
+ public static CharacterBuffer newCharacterBuffer(final int bufferSize) {
+ if(bufferSize < 2)
+ throw new IllegalArgumentException("buffersize must be >= 2");
+ return new CharacterBuffer(new char[bufferSize], 0, 0);
+ }
+ /**
+ * Fills the {@link CharacterBuffer} with characters read from the given
+ * reader {@link Reader}. This method tries to read as many characters into
+ * the {@link CharacterBuffer} as possible, each call to fill will start
+ * filling the buffer from offset 0 up to the length of the size
+ * of the internal character array.
+ *
+ * Depending on the {@link Version} passed to
+ * {@link CharacterUtils#getInstance(Version)} this method implements
+ * supplementary character awareness when filling the given buffer. For all
+ * {@link Version} > 3.0 {@link #fill(CharacterBuffer, Reader)} guarantees
+ * that the given {@link CharacterBuffer} will never contain a high surrogate
+ * character as the last element in the buffer unless it is the last available
+ * character in the reader. In other words, high and low surrogate pairs will
+ * always be preserved across buffer boarders.
+ *
+ *
+ * @param buffer
+ * the buffer to fill.
+ * @param reader
+ * the reader to read characters from.
+ * @return true if and only if no more characters are available
+ * in the reader, otherwise false.
+ * @throws IOException
+ * if the reader throws an {@link IOException}.
+ */
+ public abstract boolean fill(CharacterBuffer buffer, Reader reader) throws IOException;
+
private static final class Java5CharacterUtils extends CharacterUtils {
Java5CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -124,12 +171,32 @@
return Character.codePointAt(chars, offset, limit);
}
-
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ final char[] charBuffer = buffer.buffer;
+ buffer.offset = 0;
+ charBuffer[0] = buffer.lastTrailingHighSurrogate;
+ final int offset = buffer.lastTrailingHighSurrogate == 0 ? 0 : 1;
+ buffer.lastTrailingHighSurrogate = 0;
+ final int read = reader.read(charBuffer, offset, charBuffer.length
+ - offset);
+ if (read == -1) {
+ buffer.length = offset;
+ return offset != 0;
+ }
+ buffer.length = read + offset;
+ // special case if the read returns 0 and the lastTrailingHighSurrogate was set
+ if (buffer.length > 1
+ && Character.isHighSurrogate(charBuffer[buffer.length - 1])) {
+ buffer.lastTrailingHighSurrogate = charBuffer[--buffer.length];
+ }
+ return true;
+ }
}
private static final class Java4CharacterUtils extends CharacterUtils {
Java4CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -148,6 +215,72 @@
return chars[offset];
}
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ buffer.offset = 0;
+ final int read = reader.read(buffer.buffer);
+ if(read == -1)
+ return false;
+ buffer.length = read;
+ return true;
+ }
+
}
+
+ /**
+ * A simple IO buffer to use with
+ * {@link CharacterUtils#fill(CharacterBuffer, Reader)}.
+ */
+ public static final class CharacterBuffer {
+
+ private final char[] buffer;
+ private int offset;
+ private int length;
+ private char lastTrailingHighSurrogate = 0;
+
+ CharacterBuffer(char[] buffer, int offset, int length) {
+ this.buffer = buffer;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ /**
+ * Returns the internal buffer
+ *
+ * @return the buffer
+ */
+ public char[] getBuffer() {
+ return buffer;
+ }
+
+ /**
+ * Returns the data offset in the internal buffer.
+ *
+ * @return the offset
+ */
+ public int getOffset() {
+ return offset;
+ }
+
+ /**
+ * Return the length of the data in the internal buffer starting at
+ * {@link #getOffset()}
+ *
+ * @return the length
+ */
+ public int getLength() {
+ return length;
+ }
+
+ /**
+ * Resets the CharacterBuffer. All internals are reset to its default
+ * values.
+ */
+ public void reset() {
+ offset = 0;
+ length = 0;
+ lastTrailingHighSurrogate = 0;
+ }
+ }
}
Index: src/java/org/apache/lucene/util/Parameter.java
===================================================================
--- src/java/org/apache/lucene/util/Parameter.java (revision 916567)
+++ src/java/org/apache/lucene/util/Parameter.java (working copy)
@@ -26,6 +26,7 @@
/**
* A serializable Enum class.
* @deprecated Use Java 5 enum, will be removed in a later Lucene 3.x release.
+ * @lucene.internal
*/
@Deprecated
@SuppressWarnings("serial")
Index: src/java/org/apache/lucene/util/cache/SimpleLRUCache.java
===================================================================
--- src/java/org/apache/lucene/util/cache/SimpleLRUCache.java (revision 916567)
+++ src/java/org/apache/lucene/util/cache/SimpleLRUCache.java (working copy)
@@ -27,6 +27,8 @@
*
* @deprecated Lucene's internal use of this class has now
* switched to {@link DoubleBarrelLRUCache}.
+ *
+ * @lucene.internal
*/
@Deprecated
public class SimpleLRUCache extends SimpleMapCache {
Index: src/java/org/apache/lucene/util/cache/Cache.java
===================================================================
--- src/java/org/apache/lucene/util/cache/Cache.java (revision 916567)
+++ src/java/org/apache/lucene/util/cache/Cache.java (working copy)
@@ -21,6 +21,8 @@
/**
* Base class for cache implementations.
+ *
+ * @lucene.internal
*/
public abstract class Cache implements Closeable {
Index: src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java
===================================================================
--- src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java (revision 916567)
+++ src/java/org/apache/lucene/util/cache/DoubleBarrelLRUCache.java (working copy)
@@ -38,11 +38,7 @@
* not grow unbounded in size when under hideously high
* load.
*
- * NOTE: this class is meant only to be used internally
- * by Lucene; it's only public so it can be shared across
- * packages. This means the API is freely subject to
- * change, and, the class could be removed entirely, in any
- * Lucene release. Use directly at your own risk!
+ * @lucene.internal
*/
final public class DoubleBarrelLRUCache extends Cache {
Index: src/java/org/apache/lucene/util/cache/SimpleMapCache.java
===================================================================
--- src/java/org/apache/lucene/util/cache/SimpleMapCache.java (revision 916567)
+++ src/java/org/apache/lucene/util/cache/SimpleMapCache.java (working copy)
@@ -28,6 +28,8 @@
*
* @deprecated Lucene's internal use of this class has now
* switched to {@link DoubleBarrelLRUCache}.
+ *
+ * @lucene.internal
*/
@Deprecated
public class SimpleMapCache extends Cache {
Index: src/java/org/apache/lucene/util/Version.java
===================================================================
--- src/java/org/apache/lucene/util/Version.java (revision 916567)
+++ src/java/org/apache/lucene/util/Version.java (working copy)
@@ -50,20 +50,32 @@
/** Match settings and bugs in Lucene's 3.0 release. */
LUCENE_30,
- /** Match settings and bugs in Lucene's 3.1 release. */
+ /** Match settings and bugs in Lucene's 3.1 release.
+ *
+ * Use this to get the latest & greatest settings, bug
+ * fixes, etc, for Lucene.
+ */
LUCENE_31,
/* Add new constants for later versions **here** to respect order! */
- /** Use this to get the latest & greatest settings, bug
- * fixes, etc, for Lucene.
- *
+ /**
*
WARNING: if you use this setting, and then
* upgrade to a newer release of Lucene, sizable changes
- * may happen. If precise back compatibility is important
+ * may happen. If backwards compatibility is important
* then you should instead explicitly specify an actual
* version.
+ *
+ * If you use this constant then you may need to
+ * re-index all of your documents when upgrading
+ * Lucene, as the way text is indexed may have changed.
+ * Additionally, you may need to re-test your entire
+ * application to ensure it behaves as expected, as
+ * some defaults may have changed and may break functionality
+ * in your application.
+ * @deprecated Use an actual version instead.
*/
+ @Deprecated
LUCENE_CURRENT;
public boolean onOrAfter(Version other) {
Index: src/java/org/apache/lucene/util/SimpleStringInterner.java
===================================================================
--- src/java/org/apache/lucene/util/SimpleStringInterner.java (revision 916567)
+++ src/java/org/apache/lucene/util/SimpleStringInterner.java (working copy)
@@ -19,7 +19,10 @@
/**
* Simple lockless and memory barrier free String intern cache that is guaranteed
- * to return the same String instance as String.intern() does.
+ * to return the same String instance as String.intern()
+ * does.
+ *
+ * @lucene.internal
*/
public class SimpleStringInterner extends StringInterner {
Index: src/java/org/apache/lucene/util/VirtualMethod.java
===================================================================
--- src/java/org/apache/lucene/util/VirtualMethod.java (revision 916567)
+++ src/java/org/apache/lucene/util/VirtualMethod.java (working copy)
@@ -20,7 +20,7 @@
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashSet;
-import java.util.IdentityHashMap;
+import java.util.WeakHashMap;
import java.util.Set;
/**
@@ -54,6 +54,8 @@
* The one with the larger distance should be used preferable.
* This way also more complicated method rename scenarios can be handled
* (think of 2.9 {@code TokenStream} deprecations).
+ *
+ * @lucene.internal
*/
public final class VirtualMethod {
@@ -62,8 +64,8 @@
private final Class baseClass;
private final String method;
private final Class[] parameters;
- private final IdentityHashMap, Integer> cache =
- new IdentityHashMap, Integer>();
+ private final WeakHashMap, Integer> cache =
+ new WeakHashMap, Integer>();
/**
* Creates a new instance for the given {@code baseClass} and method declaration.
Index: src/java/org/apache/lucene/util/CloseableThreadLocal.java
===================================================================
--- src/java/org/apache/lucene/util/CloseableThreadLocal.java (revision 916567)
+++ src/java/org/apache/lucene/util/CloseableThreadLocal.java (working copy)
@@ -48,6 +48,8 @@
* {@link #close} takes care of all
* threads. You should not call {@link #close} until all
* threads are done using the instance.
+ *
+ * @lucene.internal
*/
public class CloseableThreadLocal implements Closeable {
Index: src/java/org/apache/lucene/util/ReaderUtil.java
===================================================================
--- src/java/org/apache/lucene/util/ReaderUtil.java (revision 916567)
+++ src/java/org/apache/lucene/util/ReaderUtil.java (working copy)
@@ -27,6 +27,7 @@
/**
* Common util methods for dealing with {@link IndexReader}s.
*
+ * @lucene.internal
*/
public class ReaderUtil {
Index: src/java/org/apache/lucene/util/MapOfSets.java
===================================================================
--- src/java/org/apache/lucene/util/MapOfSets.java (revision 916567)
+++ src/java/org/apache/lucene/util/MapOfSets.java (working copy)
@@ -25,6 +25,7 @@
/**
* Helper class for keeping Lists of Objects associated with keys. WARNING: THIS CLASS IS NOT THREAD SAFE
+ * @lucene.internal
*/
public class MapOfSets {
Index: src/java/org/apache/lucene/util/DummyConcurrentLock.java
===================================================================
--- src/java/org/apache/lucene/util/DummyConcurrentLock.java (revision 916567)
+++ src/java/org/apache/lucene/util/DummyConcurrentLock.java (working copy)
@@ -24,6 +24,7 @@
/**
* A dummy lock as a replacement for {@link ReentrantLock} to disable locking
+ * @lucene.internal
*/
public final class DummyConcurrentLock implements Lock {
Index: src/java/org/apache/lucene/util/IndexableBinaryStringTools.java
===================================================================
--- src/java/org/apache/lucene/util/IndexableBinaryStringTools.java (revision 916567)
+++ src/java/org/apache/lucene/util/IndexableBinaryStringTools.java (working copy)
@@ -50,6 +50,8 @@
* WARNING: This means that the deprecated Buffer-based methods
* only work correctly with buffers that have an offset of 0. For example, they
* will not correctly interpret buffers returned by {@link ByteBuffer#slice}.
+ *
+ * @lucene.experimental
*/
public class IndexableBinaryStringTools {
Index: src/java/org/apache/lucene/util/ArrayUtil.java
===================================================================
--- src/java/org/apache/lucene/util/ArrayUtil.java (revision 916567)
+++ src/java/org/apache/lucene/util/ArrayUtil.java (working copy)
@@ -7,9 +7,9 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,8 +19,20 @@
/**
* Methods for manipulating arrays.
+ *
+ * @lucene.internal
*/
public final class ArrayUtil {
+
+ /**
+ * @deprecated This constructor was not intended to be public and should not be used.
+ * This class contains solely a static utility methods.
+ * It will be made private in Lucene 4.0
+ */
+ // make private in 4.0!
+ @Deprecated
+ public ArrayUtil() {} // no instance
+
/*
Begin Apache Harmony code
@@ -122,20 +134,95 @@
END APACHE HARMONY CODE
*/
+ /** Returns an array size >= minTargetSize, generally
+ * over-allocating exponentially to achieve amortized
+ * linear-time cost as the array grows.
+ *
+ * NOTE: this was originally borrowed from Python 2.4.2
+ * listobject.c sources (attribution in LICENSE.txt), but
+ * has now been substantially changed based on
+ * discussions from java-dev thread with subject "Dynamic
+ * array reallocation algorithms", started on Jan 12
+ * 2010.
+ *
+ * @param minTargetSize Minimum required value to be returned.
+ * @param bytesPerElement Bytes used by each element of
+ * the array. See constants in {@link RamUsageEstimator}.
+ *
+ * @lucene.internal
+ */
- public static int getNextSize(int targetSize) {
- /* This over-allocates proportional to the list size, making room
- * for additional growth. The over-allocation is mild, but is
- * enough to give linear-time amortized behavior over a long
- * sequence of appends() in the presence of a poorly-performing
- * system realloc().
- * The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ...
- */
- return (targetSize >> 3) + (targetSize < 9 ? 3 : 6) + targetSize;
+ public static int oversize(int minTargetSize, int bytesPerElement) {
+
+ if (minTargetSize < 0) {
+ // catch usage that accidentally overflows int
+ throw new IllegalArgumentException("invalid array size " + minTargetSize);
+ }
+
+ if (minTargetSize == 0) {
+ // wait until at least one element is requested
+ return 0;
+ }
+
+ // asymptotic exponential growth by 1/8th, favors
+ // spending a bit more CPU to not tye up too much wasted
+ // RAM:
+ int extra = minTargetSize >> 3;
+
+ if (extra < 3) {
+ // for very small arrays, where constant overhead of
+ // realloc is presumably relatively high, we grow
+ // faster
+ extra = 3;
+ }
+
+ int newSize = minTargetSize + extra;
+
+ // add 7 to allow for worst case byte alignment addition below:
+ if (newSize+7 < 0) {
+ // int overflowed -- return max allowed array size
+ return Integer.MAX_VALUE;
+ }
+
+ if (Constants.JRE_IS_64BIT) {
+ // round up to 8 byte alignment in 64bit env
+ switch(bytesPerElement) {
+ case 4:
+ // round up to multiple of 2
+ return (newSize + 1) & 0x7ffffffe;
+ case 2:
+ // round up to multiple of 4
+ return (newSize + 3) & 0x7ffffffc;
+ case 1:
+ // round up to multiple of 8
+ return (newSize + 7) & 0x7ffffff8;
+ case 8:
+ // no rounding
+ default:
+ // odd (invalid?) size
+ return newSize;
+ }
+ } else {
+ // round up to 4 byte alignment in 64bit env
+ switch(bytesPerElement) {
+ case 2:
+ // round up to multiple of 2
+ return (newSize + 1) & 0x7ffffffe;
+ case 1:
+ // round up to multiple of 4
+ return (newSize + 3) & 0x7ffffffc;
+ case 4:
+ case 8:
+ // no rounding
+ default:
+ // odd (invalid?) size
+ return newSize;
+ }
+ }
}
- public static int getShrinkSize(int currentSize, int targetSize) {
- final int newSize = getNextSize(targetSize);
+ public static int getShrinkSize(int currentSize, int targetSize, int bytesPerElement) {
+ final int newSize = oversize(targetSize, bytesPerElement);
// Only reallocate if we are "substantially" smaller.
// This saves us from "running hot" (constantly making a
// bit bigger then a bit smaller, over and over):
@@ -147,7 +234,7 @@
public static int[] grow(int[] array, int minSize) {
if (array.length < minSize) {
- int[] newArray = new int[getNextSize(minSize)];
+ int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -159,7 +246,7 @@
}
public static int[] shrink(int[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_INT);
if (newSize != array.length) {
int[] newArray = new int[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
@@ -170,7 +257,7 @@
public static long[] grow(long[] array, int minSize) {
if (array.length < minSize) {
- long[] newArray = new long[getNextSize(minSize)];
+ long[] newArray = new long[oversize(minSize, RamUsageEstimator.NUM_BYTES_LONG)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -182,7 +269,7 @@
}
public static long[] shrink(long[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_LONG);
if (newSize != array.length) {
long[] newArray = new long[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
@@ -193,7 +280,7 @@
public static byte[] grow(byte[] array, int minSize) {
if (array.length < minSize) {
- byte[] newArray = new byte[getNextSize(minSize)];
+ byte[] newArray = new byte[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -204,10 +291,10 @@
return grow(array, 1 + array.length);
}
- public static char[] shrink(char[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ public static byte[] shrink(byte[] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
- char[] newArray = new char[newSize];
+ byte[] newArray = new byte[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
@@ -216,7 +303,7 @@
public static char[] grow(char[] array, int minSize) {
if (array.length < minSize) {
- char[] newArray = new char[getNextSize(minSize)];
+ char[] newArray = new char[oversize(minSize, RamUsageEstimator.NUM_BYTES_CHAR)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
@@ -227,10 +314,10 @@
return grow(array, 1 + array.length);
}
- public static byte[] shrink(byte[] array, int targetSize) {
- final int newSize = getShrinkSize(array.length, targetSize);
+ public static char[] shrink(char[] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_CHAR);
if (newSize != array.length) {
- byte[] newArray = new byte[newSize];
+ char[] newArray = new char[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
Index: src/java/org/apache/lucene/util/OpenBitSet.java
===================================================================
--- src/java/org/apache/lucene/util/OpenBitSet.java (revision 916567)
+++ src/java/org/apache/lucene/util/OpenBitSet.java (working copy)
@@ -73,8 +73,6 @@
1% full | 2.51 | 3.49 | | 1.00 | | 1.02 |
-
- * @version $Id$
*/
public class OpenBitSet extends DocIdSet implements Bits, Cloneable, Serializable {
@@ -809,12 +807,16 @@
@Override
public int hashCode() {
- long h = 0x98761234; // something non-zero for length==0
- for (int i = bits.length; --i>=0;) {
+ // Start with a zero hash and use a mix that results in zero if the input is zero.
+ // This effectively truncates trailing zeros without an explicit check.
+ long h = 0;
+ for (int i = bits.length; --i>=0;) {
h ^= bits[i];
h = (h << 1) | (h >>> 63); // rotate left
}
- return (int)((h>>32) ^ h); // fold leftmost bits into right
+ // fold leftmost bits into right and add a constant to prevent
+ // empty sets from returning 0, which is too common.
+ return (int)((h>>32) ^ h) + 0x98761234;
}
}
Index: src/java/org/apache/lucene/util/NumericUtils.java
===================================================================
--- src/java/org/apache/lucene/util/NumericUtils.java (revision 916567)
+++ src/java/org/apache/lucene/util/NumericUtils.java (working copy)
@@ -55,8 +55,7 @@
* {@link String#compareTo(String)}) representations of numeric data types for other
* usages (e.g. sorting).
*
- * NOTE: This API is experimental and
- * might change in incompatible ways in the next release.
+ * @lucene.experimental
*
* @since 2.9
*/
Index: src/java/org/apache/lucene/util/AttributeSource.java
===================================================================
--- src/java/org/apache/lucene/util/AttributeSource.java (revision 916567)
+++ src/java/org/apache/lucene/util/AttributeSource.java (working copy)
@@ -17,11 +17,12 @@
* limitations under the License.
*/
+import java.lang.ref.WeakReference;
import java.util.Collections;
import java.util.NoSuchElementException;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.IdentityHashMap;
+import java.util.WeakHashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
@@ -54,8 +55,8 @@
public static final AttributeFactory DEFAULT_ATTRIBUTE_FACTORY = new DefaultAttributeFactory();
private static final class DefaultAttributeFactory extends AttributeFactory {
- private static final IdentityHashMap, Class> attClassImplMap =
- new IdentityHashMap, Class>();
+ private static final WeakHashMap, WeakReference>> attClassImplMap =
+ new WeakHashMap, WeakReference>>();
private DefaultAttributeFactory() {}
@@ -72,12 +73,15 @@
private static Class getClassForInterface(Class attClass) {
synchronized(attClassImplMap) {
- Class clazz = attClassImplMap.get(attClass);
+ final WeakReference> ref = attClassImplMap.get(attClass);
+ Class clazz = (ref == null) ? null : ref.get();
if (clazz == null) {
try {
attClassImplMap.put(attClass,
- clazz = Class.forName(attClass.getName() + "Impl", true, attClass.getClassLoader())
- .asSubclass(AttributeImpl.class)
+ new WeakReference>(
+ clazz = Class.forName(attClass.getName() + "Impl", true, attClass.getClassLoader())
+ .asSubclass(AttributeImpl.class)
+ )
);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Could not find implementing class for " + attClass.getName());
@@ -173,8 +177,8 @@
}
/** a cache that stores all interfaces for known implementation classes for performance (slow reflection) */
- private static final IdentityHashMap,LinkedList>> knownImplClasses =
- new IdentityHashMap,LinkedList>>();
+ private static final WeakHashMap,LinkedList>>> knownImplClasses =
+ new WeakHashMap,LinkedList>>>();
/** Expert: Adds a custom AttributeImpl instance with one or more Attribute interfaces.
* Please note: It is not guaranteed, that att is added to
@@ -187,18 +191,20 @@
public void addAttributeImpl(final AttributeImpl att) {
final Class clazz = att.getClass();
if (attributeImpls.containsKey(clazz)) return;
- LinkedList> foundInterfaces;
+ LinkedList>> foundInterfaces;
synchronized(knownImplClasses) {
foundInterfaces = knownImplClasses.get(clazz);
if (foundInterfaces == null) {
- knownImplClasses.put(clazz, foundInterfaces = new LinkedList>());
+ // we have a strong reference to the class instance holding all interfaces in the list (parameter "att"),
+ // so all WeakReferences are never evicted by GC
+ knownImplClasses.put(clazz, foundInterfaces = new LinkedList>>());
// find all interfaces that this attribute instance implements
// and that extend the Attribute interface
Class actClazz = clazz;
do {
for (Class curInterface : actClazz.getInterfaces()) {
if (curInterface != Attribute.class && Attribute.class.isAssignableFrom(curInterface)) {
- foundInterfaces.add(curInterface.asSubclass(Attribute.class));
+ foundInterfaces.add(new WeakReference>(curInterface.asSubclass(Attribute.class)));
}
}
actClazz = actClazz.getSuperclass();
@@ -207,7 +213,10 @@
}
// add all interfaces of this AttributeImpl to the maps
- for (Class curInterface : foundInterfaces) {
+ for (WeakReference> curInterfaceRef : foundInterfaces) {
+ final Class curInterface = curInterfaceRef.get();
+ assert (curInterface != null) :
+ "We have a strong reference on the class holding the interfaces, so they should never get evicted";
// Attribute is a superclass of this interface
if (!attributes.containsKey(curInterface)) {
// invalidate state to force recomputation in captureState()
Index: src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
===================================================================
--- src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (revision 916567)
+++ src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (working copy)
@@ -44,11 +44,7 @@
* Unit tests) to check at run time if the FieldCache contains "insane"
* usages of the FieldCache.
*
- *
- * EXPERIMENTAL API: This API is considered extremely advanced and
- * experimental. It may be removed or altered w/o warning in future releases
- * of Lucene.
- *
+ * @lucene.experimental
* @see FieldCache
* @see FieldCacheSanityChecker.Insanity
* @see FieldCacheSanityChecker.InsanityType
Index: src/java/org/apache/lucene/util/UnicodeUtil.java
===================================================================
--- src/java/org/apache/lucene/util/UnicodeUtil.java (revision 916567)
+++ src/java/org/apache/lucene/util/UnicodeUtil.java (working copy)
@@ -54,8 +54,7 @@
* without always allocating a new byte[] as
* String.getBytes("UTF-8") does.
*
- * WARNING: This API is a new and experimental and
- * may suddenly change.
+ * @lucene.internal
*/
final public class UnicodeUtil {
@@ -72,6 +71,9 @@
private static final long HALF_SHIFT = 10;
private static final long HALF_MASK = 0x3FFL;
+ /**
+ * @lucene.internal
+ */
public static final class UTF16Result {
public char[] result = new char[10];
public int[] offsets = new int[10];
@@ -113,7 +115,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, RamUsageEstimator.NUM_BYTES_CHAR)];
result.offset = 0;
while(i < end) {
@@ -169,7 +171,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, RamUsageEstimator.NUM_BYTES_CHAR)];
result.offset = 0;
while(i < end) {
@@ -223,7 +225,7 @@
// Pre-allocate for worst case 4-for-1
final int maxLen = length * 4;
if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
+ out = result.bytes = new byte[ArrayUtil.oversize(maxLen, RamUsageEstimator.NUM_BYTES_CHAR)];
int upto = 0;
for(int i=offset;i= out.length) {
- char[] newOut = new char[2*(outUpto+length)];
- System.arraycopy(out, 0, newOut, 0, outUpto);
- result.result = out = newOut;
+ out = result.result = ArrayUtil.grow(out, outUpto+length+1);
}
while (upto < end) {
Index: src/java/org/apache/lucene/util/PriorityQueue.java
===================================================================
--- src/java/org/apache/lucene/util/PriorityQueue.java (revision 916567)
+++ src/java/org/apache/lucene/util/PriorityQueue.java (working copy)
@@ -23,7 +23,8 @@
*
* NOTE: This class pre-allocates a full array of
* length maxSize+1, in {@link #initialize}.
- *
+ *
+ * @lucene.internal
*/
public abstract class PriorityQueue {
private int size;
Index: src/java/org/apache/lucene/util/OpenBitSetIterator.java
===================================================================
--- src/java/org/apache/lucene/util/OpenBitSetIterator.java (revision 916567)
+++ src/java/org/apache/lucene/util/OpenBitSetIterator.java (working copy)
@@ -22,8 +22,6 @@
/** An iterator to iterate over set bits in an OpenBitSet.
* This is faster than nextSetBit() for iterating over the complete set of bits,
* especially when the density of the bits set is high.
- *
- * @version $Id$
*/
public class OpenBitSetIterator extends DocIdSetIterator {
Index: src/java/org/apache/lucene/util/ScorerDocQueue.java
===================================================================
--- src/java/org/apache/lucene/util/ScorerDocQueue.java (revision 916567)
+++ src/java/org/apache/lucene/util/ScorerDocQueue.java (working copy)
@@ -27,6 +27,8 @@
/** A ScorerDocQueue maintains a partial ordering of its Scorers such that the
least Scorer can always be found in constant time. Put()'s and pop()'s
require log(size) time. The ordering is by Scorer.doc().
+ *
+ * @lucene.internal
*/
public class ScorerDocQueue { // later: SpansQueue for spans with doc and term positions
private final HeapedScorerDoc[] heap;
Index: src/java/org/apache/lucene/util/RamUsageEstimator.java
===================================================================
--- src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 916567)
+++ src/java/org/apache/lucene/util/RamUsageEstimator.java (working copy)
@@ -30,9 +30,11 @@
* Internally uses a Map to temporally hold a reference to every
* object seen.
*
- * If checkIntered, all Strings checked will be interned, but those
+ * If checkInterned, all Strings checked will be interned, but those
* that were not already interned will be released for GC when the
* estimate is complete.
+ *
+ * @lucene.internal
*/
public final class RamUsageEstimator {
private MemoryModel memoryModel;
@@ -43,6 +45,14 @@
private int arraySize;
private int classSize;
+ public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4;
+ public final static int NUM_BYTES_CHAR = 2;
+ public final static int NUM_BYTES_SHORT = 2;
+ public final static int NUM_BYTES_INT = 4;
+ public final static int NUM_BYTES_LONG = 8;
+ public final static int NUM_BYTES_FLOAT = 4;
+ public final static int NUM_BYTES_DOUBLE = 8;
+
private boolean checkInterned;
/**
Index: src/java/org/apache/lucene/util/SortedVIntList.java
===================================================================
--- src/java/org/apache/lucene/util/SortedVIntList.java (revision 916567)
+++ src/java/org/apache/lucene/util/SortedVIntList.java (working copy)
@@ -128,8 +128,8 @@
}
if ((lastBytePos + MAX_BYTES_PER_INT) > bytes.length) {
- // biggest possible int does not fit
- resizeBytes((bytes.length * 2) + MAX_BYTES_PER_INT);
+ // Biggest possible int does not fit.
+ resizeBytes(ArrayUtil.oversize(lastBytePos + MAX_BYTES_PER_INT, 1));
}
// See org.apache.lucene.store.IndexOutput.writeVInt()
Index: src/java/org/apache/lucene/util/SmallFloat.java
===================================================================
--- src/java/org/apache/lucene/util/SmallFloat.java (revision 916567)
+++ src/java/org/apache/lucene/util/SmallFloat.java (working copy)
@@ -18,7 +18,7 @@
/** Floating point numbers smaller than 32 bits.
*
- * @version $Id$
+ * @lucene.internal
*/
public class SmallFloat {
Index: src/java/org/apache/lucene/document/NumericField.java
===================================================================
--- src/java/org/apache/lucene/document/NumericField.java (revision 916567)
+++ src/java/org/apache/lucene/document/NumericField.java (working copy)
@@ -134,9 +134,7 @@
* values are returned as {@link String}s (according to
* toString(value) of the used data type).
*
- * NOTE: This API is
- * experimental and might change in incompatible ways in the
- * next release.
+ * @lucene.experimental
*
* @since 2.9
*/
Index: src/site/src/documentation/content/xdocs/site.xml
===================================================================
--- src/site/src/documentation/content/xdocs/site.xml (revision 916567)
+++ src/site/src/documentation/content/xdocs/site.xml (working copy)
@@ -72,7 +72,6 @@
-->
-
@@ -125,7 +124,6 @@
-
Index: src/site/src/documentation/content/xdocs/lucene-contrib/index.xml
===================================================================
--- src/site/src/documentation/content/xdocs/lucene-contrib/index.xml (revision 916567)
+++ src/site/src/documentation/content/xdocs/lucene-contrib/index.xml (working copy)
@@ -120,12 +120,7 @@
-
- snowball
- Pre-compiled versions of the Snowball stemmers for Lucene.
- See snowball javadoc
-
+
spatial
Classes to help with efficient distance based sorting.
Index: src/site/src/documentation/content/xdocs/fileformats.xml
===================================================================
--- src/site/src/documentation/content/xdocs/fileformats.xml (revision 916567)
+++ src/site/src/documentation/content/xdocs/fileformats.xml (working copy)
@@ -1846,11 +1846,12 @@
When referring to term numbers, Lucene's current
- implementation uses a Java int, which means
- the maximum number of unique terms in any single index
- segment is 2,147,483,648. This is technically not a
- limitation of the index file format, just of Lucene's
- current implementation.
+ implementation uses a Java int to hold the
+ term index, which means the maximum number of unique
+ terms in any single index segment is ~2.1 billion times
+ the term index interval (default 128) = ~274 billion.
+ This is technically not a limitation of the index file
+ format, just of Lucene's current implementation.
Similarly, Lucene uses a Java int to refer
Index: docs/fileformats.pdf
===================================================================
--- docs/fileformats.pdf (revision 916567)
+++ docs/fileformats.pdf (working copy)
@@ -667,10 +667,10 @@
>>
endobj
111 0 obj
-<< /Length 1801 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 1845 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
-Gatm<>E@c%'Ro4HjK13fG\lPL)aB'A"VfQS_N=$kG,NXdH;*]f';idMD;tC&-V:5k;U.'/tp@-$LTH7^Q,1p?N)YM*ArPAGr!8(0!,&.e*ZcV$F&^;F6Ba=Bj-erc@iWHmO",>o\fW2U#KJ8%,]-0<;4\qcMW)dW(W4r')L'%gC?hm*f`HWnV%dbVZG#-_jUj#'P\O_oZ'&jIQr]%54&Rn#K1_F/7O2B740fG=lc[4Ro3YmceBq7Jn539+CM0NV+[Zoel+/JaFIhGpmP4I%\.R[ZC5-ce+e*fLJ4fAT\@3QgGc1u-7W7h"cbCre?E8TM&At8O!?\Zsm0q0)\(>&=We,.h1o`pZE7qOk1rFOCj3(%1n;J"B0i&M[BePZMTY8(,j_P+]_P5I?\J/(K8Z6DjQ_^H`SEO/Y">"_S/9Z@,81;O;nec;K.Sh35ViII!^N'h`d&Fa!oTbnAF-)SW:(h*^RW!ZkFMnf4=9B6Til(FqYcoO?tniTMb>_+7;LbV\NtQAdON4g_="n/dHr.9:I`img>pmXGCoL.5dT;_%U,hKC-2."mArlK#2QtTjDcU^OZijUGX1BS&fM_IqtMc4YW2pchD7rNX.bkiX:CO+B#XE_l!(M&o8,(lC&:VFoWIf2JGO'0#O'r5)PBdF3_]8*R*2#c0Gb-m"MMVl"4r1ri%&PJjQZZUW)F>7c#SGN$u)DUp-W7j\O%/N"t%)+rC6*J5b&i/rg?gf)qQ>0oC<>?l$4CDGTap@Wn1#_`e#([onhm(7l/DLQ].K%&"i%@qog]SY54_K@#]&\CdSEi2a6_A-\ptml007OI2_pGssbFA@6[laqdrV2`R)'V"YX4fS*\3f>,V>4#L01afe)u\m0--fn,sQ)-]FFs,o^4H.qb\/+2kb`6l+lrN;Y4Gh8JS#)F1%g,CT;[NS:Jn>_(cEbKaGProLPjEZK-n>=?sW2b%F]e/'&0&"]e\7%IOAG78TS[DN8C+WmlWBfB"=LHK?[dUc'ujV5J\KMr?G`JG;6k+/'3XN&F5`o:W;koaM4seWJhHt0u4mEMOr5Bo>0`iiqk,6HmeKLq4s5I7W&`*]&'g1*_%_OUd!0NMK,:Za1Fo%!78.6-^dpe7i'q*bg[A:V/te//^*dZeQmd3b+3">0,"]'IfX.'6n&~>
+Gatm<>BAg]'RnB3n/LcAW02O@J/Pq745JTKec65F#:5KGZ*oni-#N@,hi;M#d?,Yd>@9L+Eb+E>^2oQXAs9^1ai=_XQ0'^&@dco^idiB%[<2Vm-.F?@O`+.)LNo/%e)!DF@c9H;iPf\X6.,28n**]:&i5-/YXjesG6chX''KWD??0@/BgBgPg/i+("o=j2NX(P0'ODl8@)CV8'q/0JJ'qT,2dZj@eRhl2)eG5hql,[U'&GA5g]tBkWd"ht*C]:.X,IJD:$Y8*\o"9@*4#=aI`*i2[+QM-hWkQX:B7;*(I#aIE4Y-7]I@hlm_I=Ys^3AIVYCG^U?e;aZ/$Qjg2,^d?/3#mjXsjHcBla:tk5Zc1Wp9=8F?5!XlfGL:G5q0gNL#ZR+WL!h;L)l;MY.r[`SLg6)`G/k_:L[KZnU*72gF,.XiE3`m5#7f1@JS.WIo@efS#26'!,qgh`+lfY'KqWP4WnNG$TdmPA&P1bNXJEpZ\B5_g1KufB?AQgcV/oL3562M1$jDUZ1WCYN83`gae1`4ETh\sT\c[KpZmarCa$tR*@=Y;>DfdgNV-pfL=BX>945KHrX\,OeJpGQrNP66?Z4=7n(UJ]F!FJr.#+@(,[$AS,(A(1%\_5*=SV2gop=(.1,XC28DF:0,;X?n:1j]'Wb.SUFW*)2i<_uNVVVkC$3J`Vrg?RV1/(db)*gG\kW84OJT(0*+D9"Le6*KPe'UFCGc%8Uf"`,Uc=9Bb\NVp6YGICZ%"(U$^I)i[3O21F*H=B\^-/',Lfd)SVJd[/QeTj/7!KC?b2Sr8-R&2$!%f9\.Z;KjY/\pF]/`6V/E$iU0aau[6U,bNl>lkro4uNKp7S4\bE2qFEc1sV3VnHsGg=oj%ebEhM2anjEk9A5(cdng-*M"4t5uKquQQ_a69:iC8#Ds(]^mtB.WYGadC0Adc)WACId3qH"_<63NQG`$BeI4^mk/[+:Hd>#e@!eWJq1$^qlA*I$M'_Vg+Q_O?7#,E2C?@^ATK_ZF85?IOS2(1O_eSX_NLU4h_NH#b`4qeD:hJW!qA#HkL"n)KBH@-.LpP>=ki2,O,5a/`l#J;k\mu>9-Z2""?/*82.L2pc7M[%J+,^%=DgZ1dXbF=T#uc3+nuNVAp&*oI>jo`AQCR[td7;TqaI0,q(ah.jdG(2!m_[[pb3OpT`#>323Wj*LEW=Faj\2j3$aQbKW?iCt:Ss^4Pb5'-*c,geVDUOr:]L=E^(LLf
endstream
endobj
112 0 obj
@@ -1206,80 +1206,80 @@
xref
0 153
0000000000 65535 f
-0000054489 00000 n
-0000054695 00000 n
-0000054788 00000 n
+0000054533 00000 n
+0000054739 00000 n
+0000054832 00000 n
0000000015 00000 n
0000000071 00000 n
0000001333 00000 n
0000001453 00000 n
0000001639 00000 n
-0000054940 00000 n
+0000054984 00000 n
0000001774 00000 n
-0000055003 00000 n
+0000055047 00000 n
0000001909 00000 n
-0000055069 00000 n
+0000055113 00000 n
0000002046 00000 n
-0000055133 00000 n
+0000055177 00000 n
0000002183 00000 n
-0000055199 00000 n
+0000055243 00000 n
0000002320 00000 n
-0000055265 00000 n
+0000055309 00000 n
0000002457 00000 n
-0000055331 00000 n
+0000055375 00000 n
0000002594 00000 n
-0000055395 00000 n
+0000055439 00000 n
0000002731 00000 n
-0000055459 00000 n
+0000055503 00000 n
0000002868 00000 n
-0000055525 00000 n
+0000055569 00000 n
0000003005 00000 n
-0000055590 00000 n
+0000055634 00000 n
0000003142 00000 n
-0000055656 00000 n
+0000055700 00000 n
0000003279 00000 n
-0000055722 00000 n
+0000055766 00000 n
0000003416 00000 n
-0000055787 00000 n
+0000055831 00000 n
0000003553 00000 n
-0000055853 00000 n
+0000055897 00000 n
0000003690 00000 n
-0000055917 00000 n
+0000055961 00000 n
0000003826 00000 n
-0000055981 00000 n
+0000056025 00000 n
0000003963 00000 n
-0000056047 00000 n
+0000056091 00000 n
0000004100 00000 n
-0000056113 00000 n
+0000056157 00000 n
0000004237 00000 n
-0000056178 00000 n
+0000056222 00000 n
0000004373 00000 n
-0000056244 00000 n
+0000056288 00000 n
0000004510 00000 n
-0000056308 00000 n
+0000056352 00000 n
0000004647 00000 n
-0000056374 00000 n
+0000056418 00000 n
0000004783 00000 n
-0000056440 00000 n
+0000056484 00000 n
0000004920 00000 n
0000005673 00000 n
0000005796 00000 n
0000005872 00000 n
-0000056504 00000 n
+0000056548 00000 n
0000006004 00000 n
-0000056570 00000 n
+0000056614 00000 n
0000006137 00000 n
-0000056634 00000 n
+0000056678 00000 n
0000006270 00000 n
-0000056699 00000 n
+0000056743 00000 n
0000006403 00000 n
-0000056764 00000 n
+0000056808 00000 n
0000006536 00000 n
-0000056829 00000 n
+0000056873 00000 n
0000006669 00000 n
-0000056894 00000 n
+0000056938 00000 n
0000006801 00000 n
-0000056959 00000 n
+0000057003 00000 n
0000006934 00000 n
0000009085 00000 n
0000009193 00000 n
@@ -1317,47 +1317,47 @@
0000043768 00000 n
0000045499 00000 n
0000045609 00000 n
-0000047504 00000 n
-0000057024 00000 n
-0000047614 00000 n
-0000047814 00000 n
-0000048032 00000 n
-0000048238 00000 n
-0000048446 00000 n
-0000048614 00000 n
-0000048814 00000 n
-0000048972 00000 n
-0000049147 00000 n
-0000049410 00000 n
-0000049651 00000 n
-0000049780 00000 n
-0000049934 00000 n
-0000050088 00000 n
-0000050232 00000 n
-0000050382 00000 n
-0000050523 00000 n
-0000050758 00000 n
-0000050953 00000 n
-0000051193 00000 n
-0000051375 00000 n
-0000051548 00000 n
-0000051751 00000 n
-0000051939 00000 n
-0000052191 00000 n
-0000052332 00000 n
-0000052541 00000 n
-0000052727 00000 n
-0000052901 00000 n
-0000053146 00000 n
-0000053337 00000 n
-0000053543 00000 n
-0000053709 00000 n
-0000053823 00000 n
-0000053934 00000 n
-0000054046 00000 n
-0000054155 00000 n
-0000054262 00000 n
-0000054379 00000 n
+0000047548 00000 n
+0000057068 00000 n
+0000047658 00000 n
+0000047858 00000 n
+0000048076 00000 n
+0000048282 00000 n
+0000048490 00000 n
+0000048658 00000 n
+0000048858 00000 n
+0000049016 00000 n
+0000049191 00000 n
+0000049454 00000 n
+0000049695 00000 n
+0000049824 00000 n
+0000049978 00000 n
+0000050132 00000 n
+0000050276 00000 n
+0000050426 00000 n
+0000050567 00000 n
+0000050802 00000 n
+0000050997 00000 n
+0000051237 00000 n
+0000051419 00000 n
+0000051592 00000 n
+0000051795 00000 n
+0000051983 00000 n
+0000052235 00000 n
+0000052376 00000 n
+0000052585 00000 n
+0000052771 00000 n
+0000052945 00000 n
+0000053190 00000 n
+0000053381 00000 n
+0000053587 00000 n
+0000053753 00000 n
+0000053867 00000 n
+0000053978 00000 n
+0000054090 00000 n
+0000054199 00000 n
+0000054306 00000 n
+0000054423 00000 n
trailer
<<
/Size 153
@@ -1365,5 +1365,5 @@
/Info 4 0 R
>>
startxref
-57078
+57122
%%EOF
Index: docs/demo.html
===================================================================
--- docs/demo.html (revision 916567)
+++ docs/demo.html (working copy)
@@ -186,9 +186,6 @@
Remote